You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/01/14 07:57:10 UTC
[01/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Repository: incubator-carbondata
Updated Branches:
refs/heads/movecorepackage [created] afa88ee65
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index 1f7f443..326bde7 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -45,10 +45,10 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory.FileType;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.impl.FileFactory.FileType;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java b/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
index 3d77e11..3afc0ae 100644
--- a/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
+++ b/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
@@ -45,7 +45,7 @@ import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.load.BlockDetails;
import org.apache.carbondata.core.load.LoadMetadataDetails;
import org.apache.carbondata.core.util.CarbonProperties;
[49/57] [abbrv] incubator-carbondata git commit: fix style
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/WriterCompressModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/WriterCompressModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/WriterCompressModel.java
new file mode 100644
index 0000000..368209f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/WriterCompressModel.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression;
+
+import org.apache.carbondata.core.util.CompressionFinder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+
+public class WriterCompressModel {
+
+ /**
+ * DataType[] variable.
+ */
+ private ValueCompressionUtil.DataType[] convertedDataType;
+ /**
+ * DataType[] variable.
+ */
+ private ValueCompressionUtil.DataType[] actualDataType;
+
+ /**
+ * maxValue
+ */
+ private Object[] maxValue;
+ /**
+ * minValue.
+ */
+ private Object[] minValue;
+
+ /**
+ * uniqueValue
+ */
+ private Object[] uniqueValue;
+ /**
+ * mantissa.
+ */
+ private int[] mantissa;
+
+ /**
+ * aggType
+ */
+ private char[] type;
+
+ /**
+ * dataTypeSelected
+ */
+ private byte[] dataTypeSelected;
+ /**
+ * unCompressValues.
+ */
+ private ValueCompressionHolder[] valueHolder;
+
+ private CompressionFinder[] compressionFinders;
+
+ /**
+ * @return the convertedDataType
+ */
+ public ValueCompressionUtil.DataType[] getConvertedDataType() {
+ return convertedDataType;
+ }
+
+ /**
+ * @param convertedDataType the convertedDataType to set
+ */
+ public void setConvertedDataType(ValueCompressionUtil.DataType[] convertedDataType) {
+ this.convertedDataType = convertedDataType;
+ }
+
+ /**
+ * @return the actualDataType
+ */
+ public ValueCompressionUtil.DataType[] getActualDataType() {
+ return actualDataType;
+ }
+
+ /**
+ * @param actualDataType
+ */
+ public void setActualDataType(ValueCompressionUtil.DataType[] actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
+ /**
+ * @return the maxValue
+ */
+ public Object[] getMaxValue() {
+ return maxValue;
+ }
+
+ /**
+ * @param maxValue the maxValue to set
+ */
+ public void setMaxValue(Object[] maxValue) {
+ this.maxValue = maxValue;
+ }
+
+ /**
+ * @return the mantissa
+ */
+ public int[] getMantissa() {
+ return mantissa;
+ }
+
+ /**
+ * @param mantissa the mantissa to set
+ */
+ public void setMantissa(int[] mantissa) {
+ this.mantissa = mantissa;
+ }
+
+ /**
+ * getUnCompressValues().
+ *
+ * @return the unCompressValues
+ */
+ public ValueCompressionHolder[] getValueCompressionHolder() {
+ return valueHolder;
+ }
+
+ /**
+ * @param valueHolder set the ValueCompressionHolder
+ */
+ public void setValueCompressionHolder(ValueCompressionHolder[] valueHolder) {
+ this.valueHolder = valueHolder;
+ }
+
+ /**
+ * getMinValue
+ *
+ * @return
+ */
+ public Object[] getMinValue() {
+ return minValue;
+ }
+
+ /**
+ * setMinValue.
+ *
+ * @param minValue
+ */
+ public void setMinValue(Object[] minValue) {
+ this.minValue = minValue;
+ }
+
+ /**
+ * @return the aggType
+ */
+ public char[] getType() {
+ return type;
+ }
+
+ /**
+ * @param type the type to set
+ */
+ public void setType(char[] type) {
+ this.type = type;
+ }
+
+ /**
+ * @return the dataTypeSelected
+ */
+ public byte[] getDataTypeSelected() {
+ return dataTypeSelected;
+ }
+
+ /**
+ * @param dataTypeSelected the dataTypeSelected to set
+ */
+ public void setDataTypeSelected(byte[] dataTypeSelected) {
+ this.dataTypeSelected = dataTypeSelected;
+ }
+
+ /**
+ * getUniqueValue
+ *
+ * @return
+ */
+ public Object[] getUniqueValue() {
+ return uniqueValue;
+ }
+
+ /**
+ * setUniqueValue
+ *
+ * @param uniqueValue
+ */
+ public void setUniqueValue(Object[] uniqueValue) {
+ this.uniqueValue = uniqueValue;
+ }
+
+ public void setCompressionFinders(CompressionFinder[] compressionFinders) {
+ this.compressionFinders = compressionFinders;
+ }
+
+ public CompressionFinder[] getCompressionFinders() {
+ return this.compressionFinders;
+ }
+
+ /**
+ * @return the compType
+ */
+ public ValueCompressionUtil.COMPRESSION_TYPE getCompType(int index) {
+ return this.compressionFinders[index].getCompType();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinByte.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinByte.java
new file mode 100644
index 0000000..e1f4e06
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinByte.java
@@ -0,0 +1,115 @@
+/*
+3 * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression.decimal;
+
+import java.math.BigDecimal;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastore.compression.Compressor;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionMaxMinByte extends ValueCompressionHolder<byte[]> {
+
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionMaxMinByte.class.getName());
+
+ /**
+ * compressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+
+ /**
+ * value.
+ */
+ protected byte[] value;
+
+ private MeasureDataChunkStore<byte[]> measureChunkStore;
+
+ /**
+ * actual data type
+ */
+ protected DataType actualDataType;
+
+ private double maxValue;
+
+ public CompressionMaxMinByte(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
+ @Override public byte[] getValue() {return this.value; }
+
+ @Override public void setValue(byte[] value) {
+ this.value = value;
+ }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_BYTE, value);
+ }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, compressedData, offset, length);
+ setUncompressedValues(value, maxValueObject);
+
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ this.value = value;
+ }
+
+ @Override public long getLongValue(int index) {
+ byte byteValue = measureChunkStore.getByte(index);
+ return (long) (maxValue - byteValue);
+ }
+
+ @Override public double getDoubleValue(int index) {
+ byte byteValue = measureChunkStore.getByte(index);
+ return (maxValue - byteValue);
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal value is not defined for CompressionMaxMinByte");
+ }
+
+ private void setUncompressedValues(byte[] data, Object maxValueObject) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_BYTE, data.length);
+ this.measureChunkStore.putData(data);
+ if (maxValueObject instanceof Long) {
+ this.maxValue = (long)maxValueObject;
+ } else {
+ this.maxValue = (double) maxValueObject;
+ }
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinDefault.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinDefault.java
new file mode 100644
index 0000000..89c0016
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinDefault.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression.decimal;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastore.compression.Compressor;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+
+public class CompressionMaxMinDefault extends ValueCompressionHolder<double[]> {
+
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionMaxMinDefault.class.getName());
+
+ /**
+ * compressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+
+ /**
+ * value.
+ */
+ private double[] value;
+
+ private MeasureDataChunkStore<double[]> measureChunkStore;
+
+ /**
+ * actual data type
+ */
+ private DataType actualDataType;
+
+ private double maxValue;
+
+ public CompressionMaxMinDefault(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
+ @Override public void setValue(double[] value) {
+ this.value = value;
+ }
+
+ @Override public double[] getValue() {return this.value; }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_DOUBLE, value);
+ }
+
+ @Override public void uncompress(DataType dataType, byte[] compressedData,
+ int offset, int length, int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, compressedData, offset, length);
+ setUncompressedValues(value, maxValueObject);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ ByteBuffer buffer = ByteBuffer.wrap(value);
+ this.value = ValueCompressionUtil.convertToDoubleArray(buffer, value.length);
+ }
+
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException(
+ "Long value is not defined for CompressionMaxMinDefault");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ double doubleValue = measureChunkStore.getDouble(index);
+ return maxValue - doubleValue;
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal value is not defined for CompressionMaxMinDefault");
+ }
+
+ private void setUncompressedValues(double[] data, Object maxValueObject) {
+ this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
+ .getMeasureDataChunkStore(DataType.DATA_DOUBLE, data.length);
+ this.measureChunkStore.putData(data);
+ if (maxValueObject instanceof Long) {
+ this.maxValue = (long) maxValueObject;
+ } else {
+ this.maxValue = (double) maxValueObject;
+ }
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinInt.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinInt.java
new file mode 100644
index 0000000..fdd11ee
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinInt.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression.decimal;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastore.compression.Compressor;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionMaxMinInt extends ValueCompressionHolder<int[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionMaxMinInt.class.getName());
+
+ /**
+ * compressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+
+ private MeasureDataChunkStore<int[]> measureChunkStore;
+
+ /**
+ * value.
+ */
+ private int[] value;
+
+ private DataType actualDataType;
+
+ private double maxValue;
+
+ public CompressionMaxMinInt(DataType actualType) {
+ this.actualDataType = actualType;
+ }
+
+ @Override public void setValue(int[] value) {
+ this.value = value;
+ }
+
+ @Override public int[] getValue() { return this.value; }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, compressedData, offset, length);
+ setUncompressedValues(value, maxValueObject);
+ }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_INT, value);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ ByteBuffer buffer = ByteBuffer.wrap(value);
+ this.value = ValueCompressionUtil.convertToIntArray(buffer, value.length);
+ }
+
+ @Override public long getLongValue(int index) {
+ int intValue = measureChunkStore.getInt(index);
+ return (long) (maxValue - intValue);
+ }
+
+ @Override public double getDoubleValue(int index) {
+ int intValue = measureChunkStore.getInt(index);
+ return maxValue - intValue;
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal value is not defined for CompressionMaxMinInt");
+ }
+
+ private void setUncompressedValues(int[] data, Object maxValueObject) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, data.length);
+ this.measureChunkStore.putData(data);
+ if (maxValueObject instanceof Long) {
+ this.maxValue = (long) maxValueObject;
+ } else {
+ this.maxValue = (double) maxValueObject;
+ }
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinLong.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinLong.java
new file mode 100644
index 0000000..b8decac
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinLong.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression.decimal;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastore.compression.Compressor;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionMaxMinLong extends ValueCompressionHolder<long[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionMaxMinLong.class.getName());
+ /**
+ * compressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+
+ private MeasureDataChunkStore<long[]> measureChunkStore;
+
+ /**
+ * value.
+ */
+ protected long[] value;
+
+ protected DataType actualDataType;
+
+ private double maxValue;
+
+ public CompressionMaxMinLong(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_LONG, value);
+ }
+
+ @Override public void setValue(long[] value) {
+ this.value = value;
+
+ }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, compressedData, offset, length);
+ setUncompressValues(value, maxValueObject);
+ }
+
+ @Override public long[] getValue() {return this.value; }
+
+ @Override public void setValueInBytes(byte[] value) {
+ ByteBuffer buffer = ByteBuffer.wrap(value);
+ this.value = ValueCompressionUtil.convertToLongArray(buffer, value.length);
+ }
+
+ @Override public long getLongValue(int index) {
+ long longValue = measureChunkStore.getLong(index);
+ return (long) maxValue - longValue;
+ }
+
+ @Override public double getDoubleValue(int index) {
+ long longValue = measureChunkStore.getLong(index);
+ return maxValue - longValue;
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal value is not defined for CompressionMaxMinLong");
+ }
+
+ private void setUncompressValues(long[] data, Object maxValueObject) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_LONG, data.length);
+ this.measureChunkStore.putData(data);
+ if (maxValueObject instanceof Long) {
+ this.maxValue = (long) maxValueObject;
+ } else {
+ this.maxValue = (double) maxValueObject;
+ }
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinShort.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinShort.java
new file mode 100644
index 0000000..812d887
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/decimal/CompressionMaxMinShort.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression.decimal;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastore.compression.Compressor;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionMaxMinShort extends ValueCompressionHolder<short[]> {
+
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionMaxMinShort.class.getName());
+
+ /**
+ * shortCompressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+
+ private MeasureDataChunkStore<short[]> measureChunkStore;
+
+ /**
+ * value.
+ */
+ private short[] value;
+
+ private DataType actualDataType;
+
+ private double maxValue;
+
+ public CompressionMaxMinShort(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
+ @Override public void setValue(short[] value) {
+ this.value = value;
+
+ }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, compressedData, offset, length);
+ setUncompressedValues(value, maxValueObject);
+ }
+
+ @Override public short[] getValue() {return this.value; }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_SHORT, value);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ ByteBuffer buffer = ByteBuffer.wrap(value);
+ this.value = ValueCompressionUtil.convertToShortArray(buffer, value.length);
+ }
+
+ @Override public long getLongValue(int index) {
+ short shortValue = measureChunkStore.getShort(index);
+ return (long) maxValue - shortValue;
+ }
+
+ @Override public double getDoubleValue(int index) {
+ short shortValue = measureChunkStore.getShort(index);
+ return maxValue - shortValue;
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal value is not defined for CompressionMaxMinShort");
+ }
+
+ private void setUncompressedValues(short[] data, Object maxValueObject) {
+ this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
+ .getMeasureDataChunkStore(DataType.DATA_SHORT, data.length);
+ this.measureChunkStore.putData(data);
+ if (maxValueObject instanceof Long) {
+ this.maxValue = (long) maxValueObject;
+ } else {
+ this.maxValue = (double) maxValueObject;
+ }
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalByte.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalByte.java
new file mode 100644
index 0000000..eb3da93
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalByte.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression.nondecimal;
+
+import java.math.BigDecimal;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastore.compression.Compressor;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNonDecimalByte extends ValueCompressionHolder<byte[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNonDecimalByte.class.getName());
+
+ /**
+ * compressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+
+ /**
+ * value.
+ */
+ private byte[] value;
+
+ private MeasureDataChunkStore<byte[]> measureChunkStore;
+
+ private double divisionFactory;
+
+ @Override public void setValue(byte[] value) {
+ this.value = value;
+ }
+
+ @Override public byte[] getValue() {return this.value; }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_BYTE, value);
+ }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] compressedData,
+ int offset, int length, int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, compressedData, offset, length);
+ setUncompressedValues(value, decimalPlaces);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ this.value = value;
+ }
+
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException(
+ "Long value is not defined for CompressionNonDecimalByte");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ return (measureChunkStore.getByte(index) / this.divisionFactory);
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal value is not defined for CompressionNonDecimalByte");
+ }
+
+ private void setUncompressedValues(byte[] data, int decimalPlaces) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_BYTE, data.length);
+ this.measureChunkStore.putData(data);
+ this.divisionFactory = Math.pow(10, decimalPlaces);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalDefault.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalDefault.java
new file mode 100644
index 0000000..cf8bda9
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalDefault.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression.nondecimal;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastore.compression.Compressor;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNonDecimalDefault extends ValueCompressionHolder<double[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNonDecimalDefault.class.getName());
+ /**
+ * doubleCompressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+ /**
+ * value.
+ */
+ private double[] value;
+
+ private MeasureDataChunkStore<double[]> measureChunkStore;
+
+ private double divisionFactory;
+
+ @Override
+ public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, compressedData, offset, length);
+ setUncompressedValues(value, decimalPlaces);
+ }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_DOUBLE, value);
+ }
+
+ @Override public void setValue(double[] value) {
+ this.value = value;
+
+ }
+
+ @Override public double[] getValue() {return this.value; }
+
+ @Override public void setValueInBytes(byte[] value) {
+ ByteBuffer buffer = ByteBuffer.wrap(value);
+ this.value = ValueCompressionUtil.convertToDoubleArray(buffer, value.length);
+ }
+
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException(
+ "Long value is not defined for CompressionNonDecimalDefault");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ return (measureChunkStore.getDouble(index) / divisionFactory);
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal value is not defined for CompressionNonDecimalDefault");
+ }
+
+ private void setUncompressedValues(double[] data, int decimalPlaces) {
+ this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
+ .getMeasureDataChunkStore(DataType.DATA_DOUBLE, data.length);
+ this.measureChunkStore.putData(data);
+ this.divisionFactory = Math.pow(10, decimalPlaces);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalInt.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalInt.java
new file mode 100644
index 0000000..29ac7d0
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalInt.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression.nondecimal;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastore.compression.Compressor;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNonDecimalInt extends ValueCompressionHolder<int[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNonDecimalInt.class.getName());
+ /**
+ * intCompressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+ /**
+ * value.
+ */
+ private int[] value;
+
+ private MeasureDataChunkStore<int[]> measureChunkStore;
+
+ private double divisionFactory;
+
+ @Override public void setValue(int[] value) {
+ this.value = value;
+ }
+
+ @Override public int[] getValue() { return this.value; }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_INT, value);
+ }
+
+ @Override public void setValueInBytes(byte[] bytesArr) {
+ ByteBuffer buffer = ByteBuffer.wrap(bytesArr);
+ this.value = ValueCompressionUtil.convertToIntArray(buffer, bytesArr.length);
+ }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, compressedData, offset, length);
+ setUncompressedValues(value, decimalPlaces);
+ }
+
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException(
+ "Long value is not defined for CompressionNonDecimalInt");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ return (measureChunkStore.getInt(index) / this.divisionFactory);
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal value is not defined for CompressionNonDecmialInt");
+ }
+
+ private void setUncompressedValues(int[] data, int decimalPlaces) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, data.length);
+ this.measureChunkStore.putData(data);
+ this.divisionFactory = Math.pow(10, decimalPlaces);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalLong.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalLong.java
new file mode 100644
index 0000000..c7c5f5f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalLong.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression.nondecimal;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastore.compression.Compressor;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNonDecimalLong extends ValueCompressionHolder<long[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNonDecimalLong.class.getName());
+
+ /**
+ * longCompressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+
+ /**
+ * value.
+ */
+ private long[] value;
+
+ private MeasureDataChunkStore<long[]> measureChunkStore;
+
+ private double divisionFactory;
+
+ @Override public void setValue(long[] value) {
+ this.value = value;
+ }
+
+ @Override public long[] getValue() { return this.value; }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_LONG, value);
+ }
+
+ @Override public void uncompress(DataType dataType, byte[] compressedData, int offset,
+ int length, int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, compressedData, offset, length);
+ setUncompressedValues(value, decimalPlaces);
+ }
+
+ @Override public void setValueInBytes(byte[] bytes) {
+ ByteBuffer buffer = ByteBuffer.wrap(bytes);
+ this.value = ValueCompressionUtil.convertToLongArray(buffer, bytes.length);
+ }
+
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException(
+ "Long value is not defined for CompressionNonDecimalLong");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ return (measureChunkStore.getLong(index) / this.divisionFactory);
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal value is not defined for CompressionNonDecimalLong");
+ }
+
+ private void setUncompressedValues(long[] data, int decimalPlaces) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_LONG, data.length);
+ this.measureChunkStore.putData(data);
+ this.divisionFactory = Math.pow(10, decimalPlaces);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinByte.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinByte.java
new file mode 100644
index 0000000..6bf7a2c
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinByte.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression.nondecimal;
+
+import java.math.BigDecimal;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastore.compression.Compressor;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNonDecimalMaxMinByte extends ValueCompressionHolder<byte[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNonDecimalMaxMinByte.class.getName());
+
+ /**
+ * compressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+
+ /**
+ * value.
+ */
+ private byte[] value;
+
+ private MeasureDataChunkStore<byte[]> measureChunkStore;
+
+ private BigDecimal maxValue;
+
+ private double divisionFactor;
+
+ @Override
+ public void uncompress(DataType dataType, byte[] compressedData,
+ int offset, int length, int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, compressedData, offset, length);
+ setUncompressedValues(value, decimalPlaces, maxValueObject );
+ }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_BYTE, value);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ this.value = value;
+ }
+
+ @Override public void setValue(byte[] value) {
+ this.value = value;
+ }
+
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException(
+ "Long value is not defined for CompressionNonDecimalMaxMinByte");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ byte byteValue = measureChunkStore.getByte(index);
+ BigDecimal diff = BigDecimal.valueOf(byteValue / this.divisionFactor);
+ return maxValue.subtract(diff).doubleValue();
+
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal value is not defined for CompressionNonDecimalMaxMinByte");
+ }
+
+ private void setUncompressedValues(byte[] data, int decimalPlaces, Object maxValueObject) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_BYTE, data.length);
+ this.measureChunkStore.putData(data);
+ this.maxValue = BigDecimal.valueOf((double) maxValueObject);
+ this.divisionFactor = Math.pow(10, decimalPlaces);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+
+ @Override public byte[] getValue() { return this.value; }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java
new file mode 100644
index 0000000..3a57ec8
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression.nondecimal;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastore.compression.Compressor;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNonDecimalMaxMinDefault extends ValueCompressionHolder<double[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNonDecimalMaxMinDefault.class.getName());
+
+ /**
+ * doubleCompressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+
+ /**
+ * value.
+ */
+ private double[] value;
+
+ private MeasureDataChunkStore<double[]> measureChunkStore;
+
+ private BigDecimal maxValue;
+
+ private double divisionFactor;
+
+ @Override public void setValue(double[] value) {
+ this.value = value;
+ }
+
+ @Override public double[] getValue() { return this.value; }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_DOUBLE, value);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ ByteBuffer buffer = ByteBuffer.wrap(value);
+ this.value = ValueCompressionUtil.convertToDoubleArray(buffer, value.length);
+ }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] compressedData,
+ int offset, int length, int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor,dataType,compressedData, offset, length);
+ setUncompressedValues(value, decimalPlaces, maxValueObject);
+ }
+
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException(
+ "Long value is not defined for CompressionNonDecimalMaxMinDefault");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ double doubleValue = measureChunkStore.getDouble(index);
+ BigDecimal diff = BigDecimal.valueOf(doubleValue / this.divisionFactor);
+ return maxValue.subtract(diff).doubleValue();
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal value is not defined for CompressionNonDecimalMaxMinDefault");
+ }
+
+ private void setUncompressedValues(double[] data, int decimalPlaces, Object maxValueObject) {
+ this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
+ .getMeasureDataChunkStore(DataType.DATA_DOUBLE, data.length);
+ this.measureChunkStore.putData(data);
+ this.maxValue = BigDecimal.valueOf((double) maxValueObject);
+ this.divisionFactor = Math.pow(10, decimalPlaces);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinInt.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinInt.java
new file mode 100644
index 0000000..0597672
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinInt.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression.nondecimal;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastore.compression.Compressor;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNonDecimalMaxMinInt extends ValueCompressionHolder<int[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNonDecimalMaxMinInt.class.getName());
+ /**
+ * intCompressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+ /**
+ * value.
+ */
+ private int[] value;
+
+ private MeasureDataChunkStore<int[]> measureChunkStore;
+
+ private double divisionFactor;
+
+ private BigDecimal maxValue;
+
+ @Override public void setValue(int[] value) {
+ this.value = value;
+ }
+
+ @Override public int[] getValue() {return this.value; }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_INT, value);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ ByteBuffer buffer = ByteBuffer.wrap(value);
+ this.value = ValueCompressionUtil.convertToIntArray(buffer, value.length);
+ }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] compressedData,
+ int offset, int length, int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, compressedData, offset, length);
+ setUncompressedValues(value, decimalPlaces, maxValueObject);
+ }
+
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException(
+ "Long value is not defined for CompressionNonDecimalMaxMinInt");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ int intValue = measureChunkStore.getInt(index);
+ BigDecimal diff = BigDecimal.valueOf(intValue / this.divisionFactor);
+ return maxValue.subtract(diff).doubleValue();
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal value is not defined for CompressionNonDecimalMaxMinInt");
+ }
+
+ private void setUncompressedValues(int[] data, int decimalPlaces, Object maxValueObject) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, data.length);
+ this.measureChunkStore.putData(data);
+ this.maxValue = BigDecimal.valueOf((double) maxValueObject);
+ this.divisionFactor = Math.pow(10, decimalPlaces);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinLong.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinLong.java
new file mode 100644
index 0000000..7cd9a30
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinLong.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression.nondecimal;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastore.compression.Compressor;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNonDecimalMaxMinLong extends ValueCompressionHolder<long[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNonDecimalMaxMinLong.class.getName());
+
+ /**
+ * longCompressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+ /**
+ * value.
+ */
+ private long[] value;
+
+ private MeasureDataChunkStore<long[]> measureChunkStore;
+
+ private BigDecimal maxValue;
+
+ private double divisionFactor;
+
+ @Override public void setValue(long[] value) {
+ this.value = value;
+
+ }
+
+ @Override public long[] getValue() { return this.value; }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] compressedData,
+ int offset, int length, int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, compressedData, offset, length);
+ setUncompressedValues(value, decimalPlaces, maxValueObject);
+ }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_LONG, value);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ ByteBuffer buff = ByteBuffer.wrap(value);
+ this.value = ValueCompressionUtil.convertToLongArray(buff, value.length);
+ }
+
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException(
+ "Long value is not defined for CompressionNonDecimalMaxMinLong");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ long longValue = measureChunkStore.getLong(index);
+ BigDecimal diff = BigDecimal.valueOf(longValue / this.divisionFactor);
+ return maxValue.subtract(diff).doubleValue();
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException("Big decimal value is not supported");
+ }
+
+ private void setUncompressedValues(long[] data, int decimalPlaces, Object maxValueObject) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_LONG, data.length);
+ this.measureChunkStore.putData(data);
+ this.maxValue = BigDecimal.valueOf((double) maxValueObject);
+ this.divisionFactor = Math.pow(10, decimalPlaces);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinShort.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinShort.java
new file mode 100644
index 0000000..a3f36c6
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalMaxMinShort.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression.nondecimal;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastore.compression.Compressor;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNonDecimalMaxMinShort extends ValueCompressionHolder<short[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNonDecimalMaxMinShort.class.getName());
+ /**
+ * compressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+ /**
+ * value.
+ */
+ private short[] value;
+
+ private MeasureDataChunkStore<short[]> measureChunkStore;
+
+ private BigDecimal maxValue;
+
+ private double divisionFactor;
+
+ @Override public void setValue(short[] value) {
+ this.value = value;
+ }
+
+ @Override public short[] getValue() { return this.value; }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] compressedData,
+ int offset, int length, int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, compressedData, offset, length);
+ setUncompressedValues(value, decimalPlaces, maxValueObject);
+ }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_SHORT, value);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ ByteBuffer buffer = ByteBuffer.wrap(value);
+ this.value = ValueCompressionUtil.convertToShortArray(buffer, value.length);
+ }
+
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException(
+ "Long value is not defined for CompressionNonDecimalMaxMinShort");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ short shortValue = measureChunkStore.getShort(index);
+ BigDecimal diff = BigDecimal.valueOf(shortValue / this.divisionFactor);
+ return maxValue.subtract(diff).doubleValue();
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException("Get big decimal value is not supported");
+ }
+
+ private void setUncompressedValues(short[] data, int decimalPlaces, Object maxValueObject) {
+ this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
+ .getMeasureDataChunkStore(DataType.DATA_SHORT, data.length);
+ this.measureChunkStore.putData(data);
+ this.maxValue = BigDecimal.valueOf((double) maxValueObject);
+ this.divisionFactor = Math.pow(10, decimalPlaces);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalShort.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalShort.java
new file mode 100644
index 0000000..b65ecae
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/nondecimal/CompressionNonDecimalShort.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression.nondecimal;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastore.compression.Compressor;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNonDecimalShort extends ValueCompressionHolder<short[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNonDecimalShort.class.getName());
+ /**
+ * shortCompressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+ /**
+ * value.
+ */
+ private short[] value;
+
+ private MeasureDataChunkStore<short[]> measureChunkStore;
+
+ private double divisionFactory;
+
+ @Override public void setValue(short[] value) {
+ this.value = value;
+ }
+
+ @Override public short[] getValue() { return this.value; }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_SHORT, value);
+ }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] compressedData,
+ int offset, int length, int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor,dataType,compressedData, offset, length);
+ setUncompressedValues(value, decimalPlaces);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ ByteBuffer buffer = ByteBuffer.wrap(value);
+ this.value = ValueCompressionUtil.convertToShortArray(buffer, value.length);
+ }
+
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException(
+ "Long value is not defined for CompressionNonDecimalShort");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ return (measureChunkStore.getShort(index) / this.divisionFactory);
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal value is not defined for CompressionNonDecimalShort");
+ }
+
+ private void setUncompressedValues(short[] data, int decimalPlaces) {
+ this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
+ .getMeasureDataChunkStore(DataType.DATA_SHORT, data.length);
+ this.measureChunkStore.putData(data);
+ this.divisionFactory = Math.pow(10, decimalPlaces);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneByte.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneByte.java
new file mode 100644
index 0000000..8161557
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneByte.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression.none;
+
+import java.math.BigDecimal;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastore.compression.Compressor;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNoneByte extends ValueCompressionHolder<byte[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNoneByte.class.getName());
+
+ /**
+ * byteCompressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+
+ /**
+ * value.
+ */
+ private byte[] value;
+
+ /**
+ * actual data type
+ */
+ private DataType actualDataType;
+
+ private MeasureDataChunkStore<byte[]> measureChunkStore;
+
+ public CompressionNoneByte(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
+ @Override public void setValue(byte[] value) {
+ this.value = value;
+ }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] data, int offset, int length,
+ int mantissa, Object maxValueObject) {
+ super.unCompress(compressor, dataType, data, offset, length);
+ setUncompressedValues(value);
+ }
+
+ @Override public byte[] getValue() { return this.value; }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_BYTE, value);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ this.value = value;
+ }
+
+ @Override public long getLongValue(int index) {
+ return measureChunkStore.getByte(index);
+ }
+
+ @Override public double getDoubleValue(int index) {
+ return measureChunkStore.getByte(index);
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal is not defined for CompressionNoneByte");
+ }
+
+ private void setUncompressedValues(byte[] data) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_BYTE, data.length);
+ this.measureChunkStore.putData(data);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneDefault.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneDefault.java
new file mode 100644
index 0000000..267710f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneDefault.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression.none;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastore.compression.Compressor;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNoneDefault extends ValueCompressionHolder<double[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNoneDefault.class.getName());
+ /**
+ * doubleCompressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+ /**
+ * value.
+ */
+ private double[] value;
+
+ private DataType actualDataType;
+
+ private MeasureDataChunkStore<double[]> measureChunkStore;
+
+ public CompressionNoneDefault(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
+ @Override public void setValue(double[] value) {this.value = value; }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] data, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, data, offset, length);
+ setUncompressedValues(value);
+ }
+
+ @Override public double[] getValue() { return this.value; }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_DOUBLE, value);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ ByteBuffer buffer = ByteBuffer.wrap(value);
+ this.value = ValueCompressionUtil.convertToDoubleArray(buffer, value.length);
+ }
+
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException(
+ "Long value is not defined for CompressionNonDefault");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ return measureChunkStore.getDouble(index);
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal is not defined for CompressionNoneDefault");
+ }
+
+ private void setUncompressedValues(double[] data) {
+ this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
+ .getMeasureDataChunkStore(DataType.DATA_DOUBLE, data.length);
+ this.measureChunkStore.putData(data);
+
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneInt.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneInt.java
new file mode 100644
index 0000000..5e5e0eb
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneInt.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression.none;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastore.compression.Compressor;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNoneInt extends ValueCompressionHolder<int[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNoneInt.class.getName());
+ /**
+ * intCompressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+ /**
+ * value.
+ */
+ private int[] value;
+
+ private DataType actualDataType;
+
+ private MeasureDataChunkStore<int[]> measureChunkStore;
+
+ public CompressionNoneInt(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
+ @Override public void setValue(int[] value) {
+ this.value = value;
+ }
+
+ @Override public int[] getValue() { return this.value; }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_INT, value);
+ }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] data, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, data, offset, length);
+ setUncompressedValues(value);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ ByteBuffer buffer = ByteBuffer.wrap(value);
+ this.value = ValueCompressionUtil.convertToIntArray(buffer, value.length);
+ }
+
+ @Override public long getLongValue(int index) {
+ return measureChunkStore.getInt(index);
+ }
+
+ @Override public double getDoubleValue(int index) {
+ return measureChunkStore.getInt(index);
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal is not defined for CompressionNoneLong");
+ }
+
+ private void setUncompressedValues(int[] data) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, data.length);
+ this.measureChunkStore.putData(data);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
[56/57] [abbrv] incubator-carbondata git commit: fix style
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/core/src/test/java/org/apache/carbondata/core/updatestatus/locks/ZooKeeperLockingTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/updatestatus/locks/ZooKeeperLockingTest.java b/core/src/test/java/org/apache/carbondata/core/updatestatus/locks/ZooKeeperLockingTest.java
deleted file mode 100644
index f0037cf..0000000
--- a/core/src/test/java/org/apache/carbondata/core/updatestatus/locks/ZooKeeperLockingTest.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.updatestatus.locks;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.ServerSocket;
-import java.util.Properties;
-
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
-import org.apache.carbondata.core.util.CarbonProperties;
-
-import mockit.NonStrictExpectations;
-import org.apache.zookeeper.server.ServerConfig;
-import org.apache.zookeeper.server.ZooKeeperServerMain;
-import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * @author Administrator
- */
-public class ZooKeeperLockingTest {
-
- int freePort;
-
- /**
- * @throws java.lang.Exception
- */
- @Before public void setUp() throws Exception {
- Properties startupProperties = new Properties();
- startupProperties.setProperty("dataDir", (new File("./target").getAbsolutePath()));
- startupProperties.setProperty("dataLogDir", (new File("./target").getAbsolutePath()));
- freePort = findFreePort();
- startupProperties.setProperty("clientPort", "" + freePort);
- QuorumPeerConfig quorumConfiguration = new QuorumPeerConfig();
- try {
- quorumConfiguration.parseProperties(startupProperties);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
-
- final ZooKeeperServerMain zooKeeperServer = new ZooKeeperServerMain();
- final ServerConfig configuration = new ServerConfig();
- configuration.readFrom(quorumConfiguration);
- new Thread() {
- public void run() {
- try {
- zooKeeperServer.runFromConfig(configuration);
- } catch (IOException e) {
- System.out.println("ZooKeeper failure");
- }
- }
- }.start();
- }
-
- /**
- * @throws java.lang.Exception
- */
- @After public void tearDown() throws Exception {
- }
-
- @Test public void testZooKeeperLockingByTryingToAcquire2Locks()
- throws IllegalArgumentException, IllegalAccessException, NoSuchFieldException,
- SecurityException {
-
- final CarbonProperties cp = CarbonProperties.getInstance();
- new NonStrictExpectations(cp) {
- {
- cp.getProperty("/CarbonLocks");
- result = "/carbontests";
- cp.getProperty("spark.deploy.zookeeper.url");
- result = "127.0.0.1:" + freePort;
- }
- };
-
- ZookeeperInit zki = ZookeeperInit.getInstance("127.0.0.1:" + freePort);
-
- CarbonTableIdentifier tableIdentifier = new CarbonTableIdentifier("dbName", "tableName", "tableId");
- ZooKeeperLocking zkl =
- new ZooKeeperLocking(tableIdentifier,
- LockUsage.METADATA_LOCK);
- Assert.assertTrue(zkl.lock());
-
- ZooKeeperLocking zk2 = new ZooKeeperLocking(
- tableIdentifier, LockUsage.METADATA_LOCK);
- Assert.assertTrue(!zk2.lock());
-
- Assert.assertTrue(zkl.unlock());
- Assert.assertTrue(zk2.lock());
- Assert.assertTrue(zk2.unlock());
- }
-
- /**
- * For finding the free port available.
- *
- * @return
- */
- private static int findFreePort() {
- ServerSocket socket = null;
- try {
- socket = new ServerSocket(0);
- socket.setReuseAddress(true);
- int port = socket.getLocalPort();
- try {
- socket.close();
- } catch (IOException e) {
- // Ignore IOException on close()
- }
- return port;
- } catch (Exception e) {
- // Ignore
- } finally {
- if (socket != null) {
- try {
- socket.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
- return 2181;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionLockTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionLockTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionLockTest.scala
index 83753c5..0462797 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionLockTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionLockTest.scala
@@ -31,7 +31,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.core.updatestatus.SegmentStatusManager
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.core.updatestatus.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
+import org.apache.carbondata.core.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
index e6922fe..4a20609 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
@@ -22,7 +22,7 @@ package org.apache.carbondata.spark.testsuite.dataretention
import java.text.SimpleDateFormat
import org.apache.carbondata.core.updatestatus.{LoadMetadataDetails, SegmentStatusManager}
-import org.apache.carbondata.core.updatestatus.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
+import org.apache.carbondata.core.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
import org.apache.commons.lang3.time.DateUtils
import org.apache.spark.sql.Row
import org.apache.spark.sql.common.util.QueryTest
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
index 0f783e8..9cbfec0 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
@@ -55,6 +55,7 @@ import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.datastore.impl.FileFactory.FileType;
+import org.apache.carbondata.core.locks.ICarbonLock;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
@@ -66,7 +67,6 @@ import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.update.CarbonUpdateUtil;
import org.apache.carbondata.core.updatestatus.LoadMetadataDetails;
import org.apache.carbondata.core.updatestatus.SegmentStatusManager;
-import org.apache.carbondata.core.updatestatus.locks.ICarbonLock;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.processing.api.dataloader.DataLoadModel;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
index ca87db9..672c461 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
@@ -39,6 +39,9 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.locks.CarbonLockFactory;
+import org.apache.carbondata.core.locks.ICarbonLock;
+import org.apache.carbondata.core.locks.LockUsage;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
@@ -51,9 +54,6 @@ import org.apache.carbondata.core.update.SegmentUpdateDetails;
import org.apache.carbondata.core.updatestatus.LoadMetadataDetails;
import org.apache.carbondata.core.updatestatus.SegmentStatusManager;
import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager;
-import org.apache.carbondata.core.updatestatus.locks.CarbonLockFactory;
-import org.apache.carbondata.core.updatestatus.locks.ICarbonLock;
-import org.apache.carbondata.core.updatestatus.locks.LockUsage;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.writer.CarbonDeleteDeltaWriterImpl;
import org.apache.carbondata.processing.model.CarbonLoadModel;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
index 77499a6..77d5784 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
@@ -36,11 +36,11 @@ import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.cache.dictionary.Dictionary
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.locks.{CarbonLockFactory, LockUsage}
import org.apache.carbondata.core.metadata.{CarbonTableIdentifier, ColumnIdentifier}
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.path.CarbonTablePath
import org.apache.carbondata.core.service.{CarbonCommonFactory, PathService}
-import org.apache.carbondata.core.updatestatus.locks.{CarbonLockFactory, LockUsage}
import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory, CarbonUtil}
import org.apache.carbondata.processing.model.CarbonLoadModel
import org.apache.carbondata.spark.load.CarbonLoaderUtil
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
index 3e4c5ac..ad97626 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
@@ -28,11 +28,11 @@ import org.apache.spark.sql.execution.command.{CompactionCallableModel, Compacti
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.locks.{CarbonLockFactory, CarbonLockUtil, LockUsage}
import org.apache.carbondata.core.metadata.{CarbonMetadata, CarbonTableIdentifier}
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.update.CarbonUpdateUtil
import org.apache.carbondata.core.updatestatus.{LoadMetadataDetails, SegmentStatusManager}
-import org.apache.carbondata.core.updatestatus.locks.{CarbonLockFactory, CarbonLockUtil, LockUsage}
import org.apache.carbondata.processing.model.{CarbonDataLoadSchema, CarbonLoadModel}
import org.apache.carbondata.spark._
import org.apache.carbondata.spark.load._
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 8fe690b..5e8af1f 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -46,7 +46,7 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.path.CarbonStorePath
import org.apache.carbondata.core.update.CarbonUpdateUtil
import org.apache.carbondata.core.updatestatus.LoadMetadataDetails
-import org.apache.carbondata.core.updatestatus.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
+import org.apache.carbondata.core.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.processing.csvreaderstep.{BlockDetails, RddInpututilsForUpdate}
import org.apache.carbondata.processing.etl.DataLoadingException
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
index 156153d..4edcd34 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
@@ -39,7 +39,7 @@ import org.apache.carbondata.core.path.{CarbonStorePath, CarbonTablePath}
import org.apache.carbondata.core.update._
import org.apache.carbondata.core.update.data.RowCountDetailsVO
import org.apache.carbondata.core.updatestatus.{SegmentStatusManager, SegmentUpdateStatusManager}
-import org.apache.carbondata.core.updatestatus.locks.{CarbonLockFactory, CarbonLockUtil, LockUsage}
+import org.apache.carbondata.core.locks.{CarbonLockFactory, CarbonLockUtil, LockUsage}
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.writer.CarbonDeleteDeltaWriterImpl
import org.apache.carbondata.processing.exception.MultipleMatchingException
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 323c090..0cca07e 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -48,7 +48,7 @@ import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.path.CarbonStorePath
import org.apache.carbondata.core.update.{CarbonUpdateUtil, TupleIdEnum}
-import org.apache.carbondata.core.updatestatus.locks.{CarbonLockFactory, LockUsage}
+import org.apache.carbondata.core.locks.{CarbonLockFactory, LockUsage}
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
import org.apache.carbondata.processing.constants.TableOptionConstant
import org.apache.carbondata.processing.etl.DataLoadingException
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
index 970a7ea..e1d3454 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
@@ -43,7 +43,7 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.path.{CarbonStorePath, CarbonTablePath}
import org.apache.carbondata.core.reader.ThriftReader
import org.apache.carbondata.core.stats.{QueryStatistic, QueryStatisticsConstants}
-import org.apache.carbondata.core.updatestatus.locks.ZookeeperInit
+import org.apache.carbondata.core.locks.ZookeeperInit
import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory, CarbonUtil}
import org.apache.carbondata.core.writer.ThriftWriter
import org.apache.carbondata.format.{SchemaEvolutionEntry, TableInfo}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index db67940..5359cd7 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -30,22 +30,22 @@ import org.apache.hadoop.conf.{Configurable, Configuration}
import org.apache.hadoop.fs.Path
import org.apache.hadoop.mapreduce.Job
import org.apache.hadoop.mapreduce.lib.input.{FileInputFormat, FileSplit}
+import org.apache.spark.{SparkEnv, SparkException}
import org.apache.spark.rdd.{DataLoadCoalescedRDD, DataLoadPartitionCoalescer, UpdateCoalescedRDD}
+import org.apache.spark.sql.{CarbonEnv, DataFrame, Row, SQLContext}
import org.apache.spark.sql.execution.command.{AlterTableModel, CompactionModel, ExecutionErrors, UpdateTableModel}
import org.apache.spark.sql.hive.DistributionUtil
-import org.apache.spark.sql.{CarbonEnv, DataFrame, Row, SQLContext}
import org.apache.spark.util.SparkUtil
-import org.apache.spark.{SparkEnv, SparkException}
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.block.{Distributable, TableBlockInfo}
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.core.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
import org.apache.carbondata.core.metadata.{CarbonTableIdentifier, ColumnarFormatVersion}
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.path.CarbonStorePath
import org.apache.carbondata.core.update.CarbonUpdateUtil
import org.apache.carbondata.core.updatestatus.LoadMetadataDetails
-import org.apache.carbondata.core.updatestatus.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.processing.csvreaderstep.{BlockDetails, RddInpututilsForUpdate}
import org.apache.carbondata.processing.etl.DataLoadingException
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
index a733291..04505b5 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
@@ -23,8 +23,8 @@ import org.apache.spark.sql.execution.command.LoadTable
import org.apache.spark.sql.types._
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.metadata.{DataType => CarbonType}
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.{DataType => CarbonType}
import org.apache.carbondata.spark.CarbonOption
class CarbonDataFrameWriter(sqlContext: SQLContext, val dataFrame: DataFrame) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonScan.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonScan.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonScan.scala
index 8f3dcdf..85046cf 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonScan.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonScan.scala
@@ -23,8 +23,6 @@ import scala.collection.mutable.ArrayBuffer
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.hive.CarbonRelation
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.scan.model._
import org.apache.carbondata.spark.CarbonFilters
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
index dd12fd4..9590c39 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
@@ -23,8 +23,8 @@ import scala.collection.mutable.{LinkedHashSet, Map}
import org.apache.spark.sql.execution.command.{BucketFields, ColumnProperty, Field, PartitionerField, TableModel}
-import org.apache.carbondata.core.metadata.DataType
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.DataType
import org.apache.carbondata.core.util.DataTypeUtil
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.util.CommonUtil
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CarbonLateDecodeStrategy.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CarbonLateDecodeStrategy.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CarbonLateDecodeStrategy.scala
index 3277c18..821ff69 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CarbonLateDecodeStrategy.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/CarbonLateDecodeStrategy.scala
@@ -33,9 +33,9 @@ import org.apache.spark.sql.optimizer.CarbonDecoderRelation
import org.apache.spark.sql.sources.{BaseRelation, Filter}
import org.apache.spark.sql.types.{AtomicType, IntegerType}
+import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.schema.BucketingInfo
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.spark.CarbonAliasDecoderRelation
import org.apache.carbondata.spark.rdd.CarbonScanRDD
import org.apache.carbondata.spark.util.CarbonScalaUtil
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 9e9ef9e..8f49806 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -37,12 +37,12 @@ import org.apache.carbondata.api.CarbonStore
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.locks.{CarbonLockFactory, LockUsage}
import org.apache.carbondata.core.metadata.{CarbonMetadata, CarbonTableIdentifier, Encoding}
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.path.CarbonStorePath
import org.apache.carbondata.core.update.{CarbonUpdateUtil, TupleIdEnum}
-import org.apache.carbondata.core.updatestatus.locks.{CarbonLockFactory, LockUsage}
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
import org.apache.carbondata.processing.constants.TableOptionConstant
import org.apache.carbondata.processing.etl.DataLoadingException
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
index 5b77249..500f818 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
@@ -18,36 +18,36 @@
package org.apache.spark.sql.hive
import java.io._
-import java.util.concurrent.atomic.AtomicLong
import java.util.{GregorianCalendar, LinkedHashSet, UUID}
+import java.util.concurrent.atomic.AtomicLong
import scala.Array.canBuildFrom
import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer
import scala.util.parsing.combinator.RegexParsers
+import org.apache.spark.sql.{RuntimeConfig, SparkSession}
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.analysis.{MultiInstanceRelation, NoSuchTableException}
import org.apache.spark.sql.catalyst.expressions.AttributeReference
import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, Statistics}
import org.apache.spark.sql.execution.command.Partitioner
import org.apache.spark.sql.types._
-import org.apache.spark.sql.{RuntimeConfig, SparkSession}
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.filesystem.CarbonFile
import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.datastore.impl.FileFactory.FileType
+import org.apache.carbondata.core.locks.ZookeeperInit
+import org.apache.carbondata.core.metadata.{CarbonMetadata, CarbonTableIdentifier, DataType => CarbonDataType}
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
-import org.apache.carbondata.core.metadata.{CarbonMetadata, CarbonTableIdentifier, DataType => CarbonDataType}
import org.apache.carbondata.core.path.{CarbonStorePath, CarbonTablePath}
import org.apache.carbondata.core.reader.ThriftReader
import org.apache.carbondata.core.stats.{QueryStatistic, QueryStatisticsConstants}
import org.apache.carbondata.core.updatestatus.SegmentStatusManager
-import org.apache.carbondata.core.updatestatus.locks.ZookeeperInit
import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory, CarbonUtil}
import org.apache.carbondata.core.writer.ThriftWriter
import org.apache.carbondata.format.{SchemaEvolutionEntry, TableInfo}
[16/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/expression/conditional/GreaterThanEqualToExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/GreaterThanEqualToExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/conditional/GreaterThanEqualToExpression.java
deleted file mode 100644
index cf73e4b..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/GreaterThanEqualToExpression.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression.conditional;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.intf.ExpressionType;
-import org.apache.carbondata.scan.filter.intf.RowIntf;
-
-public class GreaterThanEqualToExpression extends BinaryConditionalExpression {
- private static final long serialVersionUID = 4185317066280688984L;
-
- public GreaterThanEqualToExpression(Expression left, Expression right) {
- super(left, right);
- }
-
- public ExpressionResult evaluate(RowIntf value)
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ExpressionResult elRes = left.evaluate(value);
- ExpressionResult erRes = right.evaluate(value);
- ExpressionResult exprResVal1 = elRes;
- if (elRes.isNull() || erRes.isNull()) {
- elRes.set(DataType.BOOLEAN, false);
- return elRes;
- }
- if (elRes.getDataType() != erRes.getDataType()) {
- if (elRes.getDataType().getPrecedenceOrder() < erRes.getDataType().getPrecedenceOrder()) {
- exprResVal1 = erRes;
- }
-
- }
- boolean result = false;
- switch (exprResVal1.getDataType()) {
- case STRING:
- result = elRes.getString().compareTo(erRes.getString()) >= 0;
- break;
- case SHORT:
- result = elRes.getShort() >= (erRes.getShort());
- break;
- case INT:
- result = elRes.getInt() >= (erRes.getInt());
- break;
- case DOUBLE:
- result = elRes.getDouble() >= (erRes.getDouble());
- break;
- case DATE:
- case TIMESTAMP:
- result = elRes.getTime() >= (erRes.getTime());
- break;
- case LONG:
- result = elRes.getLong() >= (erRes.getLong());
- break;
- case DECIMAL:
- result = elRes.getDecimal().compareTo(erRes.getDecimal()) >= 0;
- break;
- default:
- throw new FilterUnsupportedException(
- "DataType: " + exprResVal1.getDataType() + " not supported for the filter expression");
- }
- exprResVal1.set(DataType.BOOLEAN, result);
- return exprResVal1;
- }
-
- @Override public ExpressionType getFilterExpressionType() {
- return ExpressionType.GREATERTHAN_EQUALTO;
- }
-
- @Override public String getString() {
- return "GreaterThanEqualTo(" + left.getString() + ',' + right.getString() + ')';
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/expression/conditional/GreaterThanExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/GreaterThanExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/conditional/GreaterThanExpression.java
deleted file mode 100644
index 583b334..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/GreaterThanExpression.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression.conditional;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.intf.ExpressionType;
-import org.apache.carbondata.scan.filter.intf.RowIntf;
-
-
-public class GreaterThanExpression extends BinaryConditionalExpression {
- private static final long serialVersionUID = -5319109756575539219L;
-
- public GreaterThanExpression(Expression left, Expression right) {
- super(left, right);
- }
-
- @Override public ExpressionResult evaluate(RowIntf value)
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ExpressionResult exprLeftRes = left.evaluate(value);
- ExpressionResult exprRightRes = right.evaluate(value);
- ExpressionResult val1 = exprLeftRes;
- if (exprLeftRes.isNull() || exprRightRes.isNull()) {
- exprLeftRes.set(DataType.BOOLEAN, false);
- return exprLeftRes;
- }
- if (exprLeftRes.getDataType() != exprRightRes.getDataType()) {
- if (exprLeftRes.getDataType().getPrecedenceOrder() < exprRightRes.getDataType()
- .getPrecedenceOrder()) {
- val1 = exprRightRes;
- }
-
- }
- boolean result = false;
- switch (val1.getDataType()) {
- case STRING:
- result = exprLeftRes.getString().compareTo(exprRightRes.getString()) > 0;
- break;
- case DOUBLE:
- result = exprLeftRes.getDouble() > (exprRightRes.getDouble());
- break;
- case SHORT:
- result = exprLeftRes.getShort() > (exprRightRes.getShort());
- break;
- case INT:
- result = exprLeftRes.getInt() > (exprRightRes.getInt());
- break;
- case DATE:
- case TIMESTAMP:
- result = exprLeftRes.getTime() > (exprRightRes.getTime());
- break;
- case LONG:
- result = exprLeftRes.getLong() > (exprRightRes.getLong());
- break;
- case DECIMAL:
- result = exprLeftRes.getDecimal().compareTo(exprRightRes.getDecimal()) > 0;
- break;
- default:
- throw new FilterUnsupportedException(
- "DataType: " + val1.getDataType() + " not supported for the filter expression");
- }
- val1.set(DataType.BOOLEAN, result);
- return val1;
- }
-
- @Override public ExpressionType getFilterExpressionType() {
- return ExpressionType.GREATERTHAN;
- }
-
- @Override public String getString() {
- return "GreaterThan(" + left.getString() + ',' + right.getString() + ')';
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/expression/conditional/InExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/InExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/conditional/InExpression.java
deleted file mode 100644
index d3cf0ce..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/InExpression.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression.conditional;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.intf.ExpressionType;
-import org.apache.carbondata.scan.filter.intf.RowIntf;
-
-public class InExpression extends BinaryConditionalExpression {
- private static final long serialVersionUID = -3149927446694175489L;
-
- protected transient Set<ExpressionResult> setOfExprResult;
-
- public InExpression(Expression left, Expression right) {
- super(left, right);
- }
-
- @Override public ExpressionResult evaluate(RowIntf value)
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ExpressionResult leftRsult = left.evaluate(value);
-
- if (setOfExprResult == null) {
- ExpressionResult rightRsult = right.evaluate(value);
- ExpressionResult val = null;
- setOfExprResult = new HashSet<ExpressionResult>(10);
- for (ExpressionResult expressionResVal : rightRsult.getList()) {
- if (expressionResVal.getDataType().getPrecedenceOrder() < leftRsult.getDataType()
- .getPrecedenceOrder()) {
- val = leftRsult;
- } else {
- val = expressionResVal;
- }
- switch (val.getDataType()) {
- case STRING:
- val = new ExpressionResult(val.getDataType(), expressionResVal.getString());
- break;
- case SHORT:
- val = new ExpressionResult(val.getDataType(), expressionResVal.getShort());
- break;
- case INT:
- val = new ExpressionResult(val.getDataType(), expressionResVal.getInt());
- break;
- case DOUBLE:
- val = new ExpressionResult(val.getDataType(), expressionResVal.getDouble());
- break;
- case LONG:
- val = new ExpressionResult(val.getDataType(), expressionResVal.getLong());
- break;
- case DATE:
- case TIMESTAMP:
- val = new ExpressionResult(val.getDataType(), expressionResVal.getTime());
- break;
- case DECIMAL:
- val = new ExpressionResult(val.getDataType(), expressionResVal.getDecimal());
- break;
- default:
- throw new FilterUnsupportedException(
- "DataType: " + val.getDataType() + " not supported for the filter expression");
- }
- setOfExprResult.add(val);
- }
- }
- leftRsult.set(DataType.BOOLEAN, setOfExprResult.contains(leftRsult));
- return leftRsult;
- }
-
- @Override public ExpressionType getFilterExpressionType() {
- return ExpressionType.IN;
- }
-
- @Override public String getString() {
- return "IN(" + left.getString() + ',' + right.getString() + ')';
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/expression/conditional/LessThanEqualToExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/LessThanEqualToExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/conditional/LessThanEqualToExpression.java
deleted file mode 100644
index 7d4cbed..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/LessThanEqualToExpression.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression.conditional;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.intf.ExpressionType;
-import org.apache.carbondata.scan.filter.intf.RowIntf;
-
-public class LessThanEqualToExpression extends BinaryConditionalExpression {
- private static final long serialVersionUID = 1L;
-
- public LessThanEqualToExpression(Expression left, Expression right) {
- super(left, right);
- }
-
- public ExpressionResult evaluate(RowIntf value)
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ExpressionResult elRes = left.evaluate(value);
- ExpressionResult erRes = right.evaluate(value);
- ExpressionResult exprResValue1 = elRes;
- if (elRes.isNull() || erRes.isNull()) {
- elRes.set(DataType.BOOLEAN, false);
- return elRes;
- }
- if (elRes.getDataType() != erRes.getDataType()) {
- if (elRes.getDataType().getPrecedenceOrder() < erRes.getDataType().getPrecedenceOrder()) {
- exprResValue1 = erRes;
- }
-
- }
- boolean result = false;
- switch (exprResValue1.getDataType()) {
- case STRING:
- result = elRes.getString().compareTo(erRes.getString()) <= 0;
- break;
- case SHORT:
- result = elRes.getShort() <= (erRes.getShort());
- break;
- case INT:
- result = elRes.getInt() <= (erRes.getInt());
- break;
- case DOUBLE:
- result = elRes.getDouble() <= (erRes.getDouble());
- break;
- case DATE:
- case TIMESTAMP:
- result = elRes.getTime() <= (erRes.getTime());
- break;
- case LONG:
- result = elRes.getLong() <= (erRes.getLong());
- break;
- case DECIMAL:
- result = elRes.getDecimal().compareTo(erRes.getDecimal()) <= 0;
- break;
- default:
- throw new FilterUnsupportedException("DataType: " + exprResValue1.getDataType()
- + " not supported for the filter expression");
- }
- exprResValue1.set(DataType.BOOLEAN, result);
- return exprResValue1;
- }
-
- @Override public ExpressionType getFilterExpressionType() {
- // TODO Auto-generated method stub
- return ExpressionType.LESSTHAN_EQUALTO;
- }
-
- @Override public String getString() {
- return "LessThanEqualTo(" + left.getString() + ',' + right.getString() + ')';
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/expression/conditional/LessThanExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/LessThanExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/conditional/LessThanExpression.java
deleted file mode 100644
index cb004de..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/LessThanExpression.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression.conditional;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.intf.ExpressionType;
-import org.apache.carbondata.scan.filter.intf.RowIntf;
-
-public class LessThanExpression extends BinaryConditionalExpression {
-
- private static final long serialVersionUID = 6343040416663699924L;
-
- public LessThanExpression(Expression left, Expression right) {
- super(left, right);
- }
-
- public ExpressionResult evaluate(RowIntf value)
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ExpressionResult erRes = right.evaluate(value);
- ExpressionResult elRes = left.evaluate(value);
-
- ExpressionResult val1 = elRes;
-
- boolean result = false;
-
- if (elRes.isNull() || erRes.isNull()) {
- elRes.set(DataType.BOOLEAN, false);
- return elRes;
- }
- if (elRes.getDataType() != erRes.getDataType()) {
- if (elRes.getDataType().getPrecedenceOrder() < erRes.getDataType().getPrecedenceOrder()) {
- val1 = erRes;
- }
-
- }
- switch (val1.getDataType()) {
- case STRING:
- result = elRes.getString().compareTo(erRes.getString()) < 0;
- break;
- case SHORT:
- result = elRes.getShort() < (erRes.getShort());
- break;
- case INT:
- result = elRes.getInt() < (erRes.getInt());
- break;
- case DOUBLE:
- result = elRes.getDouble() < (erRes.getDouble());
- break;
- case DATE:
- case TIMESTAMP:
- result = elRes.getTime() < (erRes.getTime());
- break;
- case LONG:
- result = elRes.getLong() < (erRes.getLong());
- break;
- case DECIMAL:
- result = elRes.getDecimal().compareTo(erRes.getDecimal()) < 0;
- break;
- default:
- throw new FilterUnsupportedException(
- "DataType: " + val1.getDataType() + " not supported for the filter expression");
- }
- val1.set(DataType.BOOLEAN, result);
- return val1;
- }
-
- @Override public ExpressionType getFilterExpressionType() {
- return ExpressionType.LESSTHAN;
- }
-
- @Override public String getString() {
- return "LessThan(" + left.getString() + ',' + right.getString() + ')';
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/expression/conditional/ListExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/ListExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/conditional/ListExpression.java
deleted file mode 100644
index e57d48a..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/ListExpression.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression.conditional;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.intf.ExpressionType;
-import org.apache.carbondata.scan.filter.intf.RowIntf;
-
-public class ListExpression extends Expression {
- private static final long serialVersionUID = 1L;
-
- public ListExpression(List<Expression> children) {
- this.children = children;
- }
-
- @Override public ExpressionResult evaluate(RowIntf value) throws FilterUnsupportedException {
- List<ExpressionResult> listOfExprRes = new ArrayList<ExpressionResult>(10);
-
- for (Expression expr : children) {
- try {
- listOfExprRes.add(expr.evaluate(value));
- } catch (FilterIllegalMemberException e) {
- continue;
- }
- }
- return new ExpressionResult(listOfExprRes);
- }
-
- @Override public ExpressionType getFilterExpressionType() {
- // TODO Auto-generated method stub
- return ExpressionType.LIST;
- }
-
- @Override public String getString() {
- // TODO Auto-generated method stub
- return null;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/expression/conditional/NotEqualsExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/NotEqualsExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/conditional/NotEqualsExpression.java
deleted file mode 100644
index ab559f8..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/NotEqualsExpression.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression.conditional;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.intf.ExpressionType;
-import org.apache.carbondata.scan.filter.intf.RowIntf;
-
-public class NotEqualsExpression extends BinaryConditionalExpression {
-
- private static final long serialVersionUID = 8684006025540863973L;
- private boolean isNotNull = false;
- public NotEqualsExpression(Expression left, Expression right, boolean isNotNull) {
- super(left, right);
- this.isNotNull = isNotNull;
- }
-
- public NotEqualsExpression(Expression left, Expression right) {
- super(left, right);
- }
-
- @Override public ExpressionResult evaluate(RowIntf value)
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ExpressionResult elRes = left.evaluate(value);
- ExpressionResult erRes = right.evaluate(value);
-
- boolean result = false;
- ExpressionResult val1 = elRes;
- ExpressionResult val2 = erRes;
- if (elRes.isNull() || erRes.isNull()) {
- if (isNotNull) {
- elRes.set(DataType.BOOLEAN, elRes.isNull() != erRes.isNull());
- } else {
- elRes.set(DataType.BOOLEAN, false);
- }
- return elRes;
- }
- //default implementation if the data types are different for the resultsets
- if (elRes.getDataType() != erRes.getDataType()) {
- // result = elRes.getString().equals(erRes.getString());
- if (elRes.getDataType().getPrecedenceOrder() < erRes.getDataType().getPrecedenceOrder()) {
- val1 = erRes;
- val2 = elRes;
- }
- }
- switch (val1.getDataType()) {
- case STRING:
- result = !val1.getString().equals(val2.getString());
- break;
- case SHORT:
- result = val1.getShort().shortValue() != val2.getShort().shortValue();
- break;
- case INT:
- result = val1.getInt().intValue() != val2.getInt().intValue();
- break;
- case DOUBLE:
- result = val1.getDouble().doubleValue() != val2.getDouble().doubleValue();
- break;
- case DATE:
- case TIMESTAMP:
- result = val1.getTime().longValue() != val2.getTime().longValue();
- break;
- case LONG:
- result = elRes.getLong().longValue() != (erRes.getLong()).longValue();
- break;
- case DECIMAL:
- result = elRes.getDecimal().compareTo(erRes.getDecimal()) != 0;
- break;
- default:
- throw new FilterUnsupportedException(
- "DataType: " + val1.getDataType() + " not supported for the filter expression");
- }
- val1.set(DataType.BOOLEAN, result);
- return val1;
- }
-
- @Override public ExpressionType getFilterExpressionType() {
- return ExpressionType.NOT_EQUALS;
- }
-
- @Override public String getString() {
- return "NotEquals(" + left.getString() + ',' + right.getString() + ')';
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/expression/conditional/NotInExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/NotInExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/conditional/NotInExpression.java
deleted file mode 100644
index 2fced3d..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/NotInExpression.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression.conditional;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.intf.ExpressionType;
-import org.apache.carbondata.scan.filter.intf.RowIntf;
-
-public class NotInExpression extends BinaryConditionalExpression {
- private static final long serialVersionUID = -6835841923752118034L;
- protected transient Set<ExpressionResult> setOfExprResult;
-
- public NotInExpression(Expression left, Expression right) {
- super(left, right);
- }
-
- @Override public ExpressionResult evaluate(RowIntf value)
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ExpressionResult leftRsult = left.evaluate(value);
- if (setOfExprResult == null) {
- ExpressionResult val = null;
- ExpressionResult rightRsult = right.evaluate(value);
- setOfExprResult = new HashSet<ExpressionResult>(10);
- for (ExpressionResult exprResVal : rightRsult.getList()) {
- if (exprResVal.getDataType().getPrecedenceOrder() < leftRsult.getDataType()
- .getPrecedenceOrder()) {
- val = leftRsult;
- } else {
- val = exprResVal;
- }
- switch (val.getDataType()) {
- case STRING:
- val = new ExpressionResult(val.getDataType(), exprResVal.getString());
- break;
- case SHORT:
- val = new ExpressionResult(val.getDataType(), exprResVal.getShort());
- break;
- case INT:
- val = new ExpressionResult(val.getDataType(), exprResVal.getInt());
- break;
- case DOUBLE:
- val = new ExpressionResult(val.getDataType(), exprResVal.getDouble());
- break;
- case DATE:
- case TIMESTAMP:
- val = new ExpressionResult(val.getDataType(), exprResVal.getTime());
- break;
- case LONG:
- val = new ExpressionResult(val.getDataType(), exprResVal.getLong());
- break;
- case DECIMAL:
- val = new ExpressionResult(val.getDataType(), exprResVal.getDecimal());
- break;
- default:
- throw new FilterUnsupportedException(
- "DataType: " + val.getDataType() + " not supported for the filter expression");
- }
- setOfExprResult.add(val);
- }
- }
- leftRsult.set(DataType.BOOLEAN, !setOfExprResult.contains(leftRsult));
-
- return leftRsult;
- }
-
- @Override public ExpressionType getFilterExpressionType() {
- return ExpressionType.NOT_IN;
- }
-
- @Override public String getString() {
- return "NOT IN(" + left.getString() + ',' + right.getString() + ')';
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/expression/exception/FilterIllegalMemberException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/exception/FilterIllegalMemberException.java b/core/src/main/java/org/apache/carbondata/scan/expression/exception/FilterIllegalMemberException.java
deleted file mode 100644
index 96b3930..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/expression/exception/FilterIllegalMemberException.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression.exception;
-
-import java.util.Locale;
-
-/**
- * FilterIllegalMemberException class representing exception which can cause while evaluating
- * filter members needs to be gracefully handled without propagating to outer layer so that
- * the execution should not get interrupted.
- */
-public class FilterIllegalMemberException extends Exception {
-
- /**
- * default serial version ID.
- */
- private static final long serialVersionUID = 1L;
-
- /**
- * The Error message.
- */
- private String msg = "";
-
- /**
- * Constructor
- *
- * @param msg The error message for this exception.
- */
- public FilterIllegalMemberException(String msg) {
- super(msg);
- this.msg = msg;
- }
-
- /**
- * Constructor
- *
- * @param msg The error message for this exception.
- */
- public FilterIllegalMemberException(String msg, Throwable t) {
- super(msg, t);
- this.msg = msg;
- }
-
- /**
- * Constructor
- *
- */
- public FilterIllegalMemberException(Throwable t) {
- super(t);
- }
-
- /**
- * This method is used to get the localized message.
- *
- * @param locale - A Locale object represents a specific geographical,
- * political, or cultural region.
- * @return - Localized error message.
- */
- public String getLocalizedMessage(Locale locale) {
- return "";
- }
-
- /**
- * getLocalizedMessage
- */
- @Override public String getLocalizedMessage() {
- return super.getLocalizedMessage();
- }
-
- /**
- * getMessage
- */
- public String getMessage() {
- return this.msg;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/expression/exception/FilterUnsupportedException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/exception/FilterUnsupportedException.java b/core/src/main/java/org/apache/carbondata/scan/expression/exception/FilterUnsupportedException.java
deleted file mode 100644
index d54ce69..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/expression/exception/FilterUnsupportedException.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression.exception;
-
-import java.util.Locale;
-
-public class FilterUnsupportedException extends Exception {
-
- /**
- * default serial version ID.
- */
- private static final long serialVersionUID = 1L;
-
- /**
- * The Error message.
- */
- private String msg = "";
-
- /**
- * Constructor
- *
- * @param msg The error message for this exception.
- */
- public FilterUnsupportedException(String msg) {
- super(msg);
- this.msg = msg;
- }
-
- /**
- * Constructor
- *
- * @param msg The error message for this exception.
- */
- public FilterUnsupportedException(String msg, Throwable t) {
- super(msg, t);
- this.msg = msg;
- }
-
- /**
- * Constructor
- *
- */
- public FilterUnsupportedException(Throwable t) {
- super(t);
- }
-
- /**
- * This method is used to get the localized message.
- *
- * @param locale - A Locale object represents a specific geographical,
- * political, or cultural region.
- * @return - Localized error message.
- */
- public String getLocalizedMessage(Locale locale) {
- return "";
- }
-
- /**
- * getLocalizedMessage
- */
- @Override public String getLocalizedMessage() {
- return super.getLocalizedMessage();
- }
-
- /**
- * getMessage
- */
- public String getMessage() {
- return this.msg;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/expression/logical/AndExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/logical/AndExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/logical/AndExpression.java
deleted file mode 100644
index 037115a..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/expression/logical/AndExpression.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression.logical;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.intf.ExpressionType;
-import org.apache.carbondata.scan.filter.intf.RowIntf;
-
-public class AndExpression extends BinaryLogicalExpression {
-
- private static final long serialVersionUID = 1L;
-
- public AndExpression(Expression left, Expression right) {
- super(left, right);
- }
-
- @Override public ExpressionResult evaluate(RowIntf value)
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ExpressionResult resultLeft = left.evaluate(value);
- ExpressionResult resultRight = right.evaluate(value);
- switch (resultLeft.getDataType()) {
- case BOOLEAN:
- resultLeft.set(DataType.BOOLEAN, (resultLeft.getBoolean() && resultRight.getBoolean()));
- break;
- default:
- throw new FilterUnsupportedException(
- "Incompatible datatype for applying AND Expression Filter");
- }
- return resultLeft;
- }
-
- @Override public ExpressionType getFilterExpressionType() {
- // TODO Auto-generated method stub
- return ExpressionType.AND;
- }
-
- @Override public String getString() {
- // TODO Auto-generated method stub
- return "And(" + left.getString() + ',' + right.getString() + ')';
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/expression/logical/BinaryLogicalExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/logical/BinaryLogicalExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/logical/BinaryLogicalExpression.java
deleted file mode 100644
index d4ad063..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/expression/logical/BinaryLogicalExpression.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression.logical;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.scan.expression.BinaryExpression;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.LiteralExpression;
-
-public abstract class BinaryLogicalExpression extends BinaryExpression {
-
- /**
- *
- */
- private static final long serialVersionUID = 1L;
-
- public BinaryLogicalExpression(Expression left, Expression right) {
- super(left, right);
- // TODO Auto-generated constructor stub
- }
-
- public List<ExpressionResult> getLiterals() {
- List<ExpressionResult> listOfExp =
- new ArrayList<ExpressionResult>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- getExpressionResultList(this, listOfExp);
- Collections.sort(listOfExp);
- return listOfExp;
- }
-
- // Will get the column informations involved in the expressions by
- // traversing the tree
- public List<ColumnExpression> getColumnList() {
- // TODO
- List<ColumnExpression> listOfExp =
- new ArrayList<ColumnExpression>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- getColumnList(this, listOfExp);
- return listOfExp;
- }
-
- private void getColumnList(Expression expression, List<ColumnExpression> lst) {
- if (expression instanceof ColumnExpression) {
- ColumnExpression colExp = (ColumnExpression) expression;
- boolean found = false;
-
- for (ColumnExpression currentColExp : lst) {
- if (currentColExp.getColumnName().equals(colExp.getColumnName())) {
- found = true;
- colExp.setColIndex(currentColExp.getColIndex());
- break;
- }
- }
- if (!found) {
- colExp.setColIndex(lst.size());
- lst.add(colExp);
- }
- }
- for (Expression child : expression.getChildren()) {
- getColumnList(child, lst);
- }
- }
-
- public boolean isSingleDimension() {
- List<ColumnExpression> listOfExp =
- new ArrayList<ColumnExpression>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- getColumnList(this, listOfExp);
- if (listOfExp.size() == 1 && listOfExp.get(0).isDimension()) {
- return true;
- }
- return false;
-
- }
-
- private void getExpressionResultList(Expression binaryConditionalExpression,
- List<ExpressionResult> listOfExp) {
- if (binaryConditionalExpression instanceof LiteralExpression) {
- ExpressionResult colExp =
- ((LiteralExpression) binaryConditionalExpression).getExpressionResult();
- listOfExp.add(colExp);
- }
- for (Expression child : binaryConditionalExpression.getChildren()) {
- getExpressionResultList(child, listOfExp);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/expression/logical/FalseExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/logical/FalseExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/logical/FalseExpression.java
deleted file mode 100644
index 4052cb8..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/expression/logical/FalseExpression.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression.logical;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.LiteralExpression;
-import org.apache.carbondata.scan.expression.conditional.BinaryConditionalExpression;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.intf.ExpressionType;
-import org.apache.carbondata.scan.filter.intf.RowIntf;
-
-
-
-/**
- * This class will form an expression whose evaluation will be always false.
- */
-public class FalseExpression extends BinaryConditionalExpression {
-
-
- private static final long serialVersionUID = -8390184061336799370L;
-
- public FalseExpression(Expression child1) {
- super(child1, new LiteralExpression(null,null));
- }
-
- /**
- * This method will always return false, mainly used in the filter expressions
- * which are illogical.
- * eg: columnName NOT IN('Java',NULL)
- * @param value
- * @return
- * @throws FilterUnsupportedException
- * @throws FilterIllegalMemberException
- */
- @Override public ExpressionResult evaluate(RowIntf value)
- throws FilterUnsupportedException, FilterIllegalMemberException {
- return new ExpressionResult(DataType.BOOLEAN,false);
- }
-
- /**
- * This method will return the expression types
- * @return
- */
- @Override public ExpressionType getFilterExpressionType() {
- return ExpressionType.FALSE;
- }
- @Override public String getString() {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/expression/logical/OrExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/logical/OrExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/logical/OrExpression.java
deleted file mode 100644
index 8c81f1d..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/expression/logical/OrExpression.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression.logical;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.intf.ExpressionType;
-import org.apache.carbondata.scan.filter.intf.RowIntf;
-
-public class OrExpression extends BinaryLogicalExpression {
-
- private static final long serialVersionUID = 4220598043176438380L;
-
- public OrExpression(Expression left, Expression right) {
- super(left, right);
- }
-
- @Override public ExpressionResult evaluate(RowIntf value)
- throws FilterIllegalMemberException, FilterUnsupportedException {
- ExpressionResult resultLeft = left.evaluate(value);
- ExpressionResult resultRight = right.evaluate(value);
- switch (resultLeft.getDataType()) {
- case BOOLEAN:
- resultLeft.set(DataType.BOOLEAN, (resultLeft.getBoolean() || resultRight.getBoolean()));
- break;
- default:
- throw new FilterUnsupportedException(
- "Incompatible datatype for applying OR Expression Filter");
- }
-
- return resultLeft;
- }
-
- @Override public ExpressionType getFilterExpressionType() {
- return ExpressionType.OR;
- }
-
- @Override public String getString() {
- return "Or(" + left.getString() + ',' + right.getString() + ')';
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/DimColumnFilterInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/DimColumnFilterInfo.java b/core/src/main/java/org/apache/carbondata/scan/filter/DimColumnFilterInfo.java
deleted file mode 100644
index f97fffc..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/DimColumnFilterInfo.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.filter;
-
-import java.io.Serializable;
-import java.util.List;
-
-public class DimColumnFilterInfo implements Serializable {
-
- private static final long serialVersionUID = 8181578747306832771L;
-
- private boolean isIncludeFilter;
-
- private List<Integer> filterList;
-
- /**
- * Implicit column filter values to be used for block and blocklet pruning
- */
- private List<String> implicitColumnFilterList;
- private List<Integer> excludeFilterList;
- /**
- * maintain the no dictionary filter values list.
- */
- private List<byte[]> noDictionaryFilterValuesList;
-
- public List<byte[]> getNoDictionaryFilterValuesList() {
- return noDictionaryFilterValuesList;
- }
-
- public boolean isIncludeFilter() {
- return isIncludeFilter;
- }
-
- public void setIncludeFilter(boolean isIncludeFilter) {
- this.isIncludeFilter = isIncludeFilter;
- }
-
- public List<Integer> getFilterList() {
- return filterList;
- }
-
- public void setFilterList(List<Integer> filterList) {
- this.filterList = filterList;
- }
-
- public void setFilterListForNoDictionaryCols(List<byte[]> noDictionaryFilterValuesList) {
- this.noDictionaryFilterValuesList = noDictionaryFilterValuesList;
- }
-
- public List<Integer> getExcludeFilterList() {
- return excludeFilterList;
- }
- public void setExcludeFilterList(List<Integer> excludeFilterList) {
- this.excludeFilterList = excludeFilterList;
- }
- public List<String> getImplicitColumnFilterList() {
- return implicitColumnFilterList;
- }
-
- public void setImplicitColumnFilterList(List<String> implicitColumnFilterList) {
- this.implicitColumnFilterList = implicitColumnFilterList;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/scan/filter/FilterExpressionProcessor.java
deleted file mode 100644
index 3ec1bc5..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/FilterExpressionProcessor.java
+++ /dev/null
@@ -1,338 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.List;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.datastore.DataRefNode;
-import org.apache.carbondata.core.datastore.DataRefNodeFinder;
-import org.apache.carbondata.core.datastore.IndexKey;
-import org.apache.carbondata.core.datastore.block.AbstractIndex;
-import org.apache.carbondata.core.datastore.impl.btree.BTreeDataRefNodeFinder;
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.core.metadata.Encoding;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.scan.expression.BinaryExpression;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.conditional.BinaryConditionalExpression;
-import org.apache.carbondata.scan.expression.conditional.ConditionalExpression;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.executer.FilterExecuter;
-import org.apache.carbondata.scan.filter.intf.ExpressionType;
-import org.apache.carbondata.scan.filter.resolver.ConditionalFilterResolverImpl;
-import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
-import org.apache.carbondata.scan.filter.resolver.LogicalFilterResolverImpl;
-import org.apache.carbondata.scan.filter.resolver.RowLevelFilterResolverImpl;
-import org.apache.carbondata.scan.filter.resolver.RowLevelRangeFilterResolverImpl;
-
-public class FilterExpressionProcessor implements FilterProcessor {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(FilterExpressionProcessor.class.getName());
-
- /**
- * Implementation will provide the resolved form of filters based on the
- * filter expression tree which is been passed in Expression instance.
- *
- * @param expressionTree , filter expression tree
- * @param tableIdentifier ,contains carbon store informations
- * @return a filter resolver tree
- */
- public FilterResolverIntf getFilterResolver(Expression expressionTree,
- AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException, IOException {
- if (null != expressionTree && null != tableIdentifier) {
- return getFilterResolvertree(expressionTree, tableIdentifier);
- }
- return null;
- }
-
- /**
- * This API will scan the Segment level all btrees and selects the required
- * block reference nodes inorder to push the same to executer for applying filters
- * on the respective data reference node.
- * Following Algorithm is followed in below API
- * Step:1 Get the start end key based on the filter tree resolver information
- * Step:2 Prepare the IndexKeys inorder to scan the tree and get the start and end reference
- * node(block)
- * Step:3 Once data reference node ranges retrieved traverse the node within this range
- * and select the node based on the block min and max value and the filter value.
- * Step:4 The selected blocks will be send to executers for applying the filters with the help
- * of Filter executers.
- *
- */
- public List<DataRefNode> getFilterredBlocks(DataRefNode btreeNode,
- FilterResolverIntf filterResolver, AbstractIndex tableSegment,
- AbsoluteTableIdentifier tableIdentifier) {
- // Need to get the current dimension tables
- List<DataRefNode> listOfDataBlocksToScan = new ArrayList<DataRefNode>();
- // getting the start and end index key based on filter for hitting the
- // selected block reference nodes based on filter resolver tree.
- LOGGER.debug("preparing the start and end key for finding"
- + "start and end block as per filter resolver");
- List<IndexKey> listOfStartEndKeys = new ArrayList<IndexKey>(2);
- FilterUtil.traverseResolverTreeAndGetStartAndEndKey(tableSegment.getSegmentProperties(),
- filterResolver, listOfStartEndKeys);
- // reading the first value from list which has start key
- IndexKey searchStartKey = listOfStartEndKeys.get(0);
- // reading the last value from list which has end key
- IndexKey searchEndKey = listOfStartEndKeys.get(1);
- if (null == searchStartKey && null == searchEndKey) {
- try {
- // TODO need to handle for no dictionary dimensions
- searchStartKey =
- FilterUtil.prepareDefaultStartIndexKey(tableSegment.getSegmentProperties());
- // TODO need to handle for no dictionary dimensions
- searchEndKey = FilterUtil.prepareDefaultEndIndexKey(tableSegment.getSegmentProperties());
- } catch (KeyGenException e) {
- return listOfDataBlocksToScan;
- }
- }
-
- LOGGER.debug(
- "Successfully retrieved the start and end key" + "Dictionary Start Key: " + searchStartKey
- .getDictionaryKeys() + "No Dictionary Start Key " + searchStartKey.getNoDictionaryKeys()
- + "Dictionary End Key: " + searchEndKey.getDictionaryKeys() + "No Dictionary End Key "
- + searchEndKey.getNoDictionaryKeys());
- long startTimeInMillis = System.currentTimeMillis();
- DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(
- tableSegment.getSegmentProperties().getEachDimColumnValueSize());
- DataRefNode startBlock = blockFinder.findFirstDataBlock(btreeNode, searchStartKey);
- DataRefNode endBlock = blockFinder.findLastDataBlock(btreeNode, searchEndKey);
- FilterExecuter filterExecuter =
- FilterUtil.getFilterExecuterTree(filterResolver, tableSegment.getSegmentProperties(),null);
- while (startBlock != endBlock) {
- addBlockBasedOnMinMaxValue(filterExecuter, listOfDataBlocksToScan, startBlock);
- startBlock = startBlock.getNextDataRefNode();
- }
- addBlockBasedOnMinMaxValue(filterExecuter, listOfDataBlocksToScan, endBlock);
- LOGGER.info("Total Time in retrieving the data reference node" + "after scanning the btree " + (
- System.currentTimeMillis() - startTimeInMillis)
- + " Total number of data reference node for executing filter(s) " + listOfDataBlocksToScan
- .size());
-
- return listOfDataBlocksToScan;
- }
-
- /**
- * Selects the blocks based on col max and min value.
- *
- * @param listOfDataBlocksToScan
- * @param dataRefNode
- */
- private void addBlockBasedOnMinMaxValue(FilterExecuter filterExecuter,
- List<DataRefNode> listOfDataBlocksToScan, DataRefNode dataRefNode) {
-
- BitSet bitSet = filterExecuter
- .isScanRequired(dataRefNode.getColumnsMaxValue(), dataRefNode.getColumnsMinValue());
- if (!bitSet.isEmpty()) {
- listOfDataBlocksToScan.add(dataRefNode);
-
- }
- }
-
- /**
- * API will return a filter resolver instance which will be used by
- * executers to evaluate or execute the filters.
- *
- * @param expressionTree , resolver tree which will hold the resolver tree based on
- * filter expression.
- * @return FilterResolverIntf type.
- */
- private FilterResolverIntf getFilterResolvertree(Expression expressionTree,
- AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException, IOException {
- FilterResolverIntf filterEvaluatorTree =
- createFilterResolverTree(expressionTree, tableIdentifier);
- traverseAndResolveTree(filterEvaluatorTree, tableIdentifier);
- return filterEvaluatorTree;
- }
-
- /**
- * constructing the filter resolver tree based on filter expression.
- * this method will visit each node of the filter resolver and prepares
- * the surrogates of the filter members which are involved filter
- * expression.
- *
- * @param filterResolverTree
- * @param tableIdentifier
- */
- private void traverseAndResolveTree(FilterResolverIntf filterResolverTree,
- AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException, IOException {
- if (null == filterResolverTree) {
- return;
- }
- traverseAndResolveTree(filterResolverTree.getLeft(), tableIdentifier);
- filterResolverTree.resolve(tableIdentifier);
- traverseAndResolveTree(filterResolverTree.getRight(), tableIdentifier);
- }
-
- /**
- * Pattern used : Visitor Pattern
- * Method will create filter resolver tree based on the filter expression tree,
- * in this algorithm based on the expression instance the resolvers will created
- *
- * @param expressionTree
- * @param tableIdentifier
- * @return
- */
- private FilterResolverIntf createFilterResolverTree(Expression expressionTree,
- AbsoluteTableIdentifier tableIdentifier) {
- ExpressionType filterExpressionType = expressionTree.getFilterExpressionType();
- BinaryExpression currentExpression = null;
- switch (filterExpressionType) {
- case OR:
- currentExpression = (BinaryExpression) expressionTree;
- return new LogicalFilterResolverImpl(
- createFilterResolverTree(currentExpression.getLeft(), tableIdentifier),
- createFilterResolverTree(currentExpression.getRight(), tableIdentifier),
- currentExpression);
- case AND:
- currentExpression = (BinaryExpression) expressionTree;
- return new LogicalFilterResolverImpl(
- createFilterResolverTree(currentExpression.getLeft(), tableIdentifier),
- createFilterResolverTree(currentExpression.getRight(), tableIdentifier),
- currentExpression);
- case EQUALS:
- case IN:
- return getFilterResolverBasedOnExpressionType(ExpressionType.EQUALS,
- ((BinaryConditionalExpression) expressionTree).isNull, expressionTree,
- tableIdentifier, expressionTree);
- case GREATERTHAN:
- case GREATERTHAN_EQUALTO:
- case LESSTHAN:
- case LESSTHAN_EQUALTO:
- return getFilterResolverBasedOnExpressionType(ExpressionType.EQUALS, true, expressionTree,
- tableIdentifier, expressionTree);
-
- case NOT_EQUALS:
- case NOT_IN:
- return getFilterResolverBasedOnExpressionType(ExpressionType.NOT_EQUALS, false,
- expressionTree, tableIdentifier, expressionTree);
- case FALSE:
- return getFilterResolverBasedOnExpressionType(ExpressionType.FALSE, false,
- expressionTree, tableIdentifier, expressionTree);
- default:
- return getFilterResolverBasedOnExpressionType(ExpressionType.UNKNOWN, false, expressionTree,
- tableIdentifier, expressionTree);
- }
- }
-
- /**
- * Factory method which will return the resolver instance based on filter expression
- * expressions.
- */
- private FilterResolverIntf getFilterResolverBasedOnExpressionType(
- ExpressionType filterExpressionType, boolean isExpressionResolve, Expression expression,
- AbsoluteTableIdentifier tableIdentifier, Expression expressionTree) {
- BinaryConditionalExpression currentCondExpression = null;
- ConditionalExpression condExpression = null;
- switch (filterExpressionType) {
- case FALSE:
- return new RowLevelFilterResolverImpl(expression, false, false, tableIdentifier);
- case EQUALS:
- currentCondExpression = (BinaryConditionalExpression) expression;
- if (currentCondExpression.isSingleDimension()
- && currentCondExpression.getColumnList().get(0).getCarbonColumn().getDataType()
- != DataType.ARRAY
- && currentCondExpression.getColumnList().get(0).getCarbonColumn().getDataType()
- != DataType.STRUCT) {
- // getting new dim index.
- if (!currentCondExpression.getColumnList().get(0).getCarbonColumn()
- .hasEncoding(Encoding.DICTIONARY) || currentCondExpression.getColumnList().get(0)
- .getCarbonColumn().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
- if (FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getLeft())
- && FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getRight()) || (
- FilterUtil.checkIfRightExpressionRequireEvaluation(currentCondExpression.getRight())
- || FilterUtil
- .checkIfLeftExpressionRequireEvaluation(currentCondExpression.getLeft()))) {
- return new RowLevelFilterResolverImpl(expression, isExpressionResolve, true,
- tableIdentifier);
- }
- if (currentCondExpression.getFilterExpressionType() == ExpressionType.GREATERTHAN
- || currentCondExpression.getFilterExpressionType() == ExpressionType.LESSTHAN
- || currentCondExpression.getFilterExpressionType()
- == ExpressionType.GREATERTHAN_EQUALTO
- || currentCondExpression.getFilterExpressionType()
- == ExpressionType.LESSTHAN_EQUALTO) {
- return new RowLevelRangeFilterResolverImpl(expression, isExpressionResolve, true,
- tableIdentifier);
- }
- }
- return new ConditionalFilterResolverImpl(expression, isExpressionResolve, true);
-
- }
- break;
- case NOT_EQUALS:
- currentCondExpression = (BinaryConditionalExpression) expression;
- if (currentCondExpression.isSingleDimension()
- && currentCondExpression.getColumnList().get(0).getCarbonColumn().getDataType()
- != DataType.ARRAY
- && currentCondExpression.getColumnList().get(0).getCarbonColumn().getDataType()
- != DataType.STRUCT) {
- if (!currentCondExpression.getColumnList().get(0).getCarbonColumn()
- .hasEncoding(Encoding.DICTIONARY) || currentCondExpression.getColumnList().get(0)
- .getCarbonColumn().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
- if (FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getLeft())
- && FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getRight()) || (
- FilterUtil.checkIfRightExpressionRequireEvaluation(currentCondExpression.getRight())
- || FilterUtil
- .checkIfLeftExpressionRequireEvaluation(currentCondExpression.getLeft()))) {
- return new RowLevelFilterResolverImpl(expression, isExpressionResolve, false,
- tableIdentifier);
- }
- if (expressionTree.getFilterExpressionType() == ExpressionType.GREATERTHAN
- || expressionTree.getFilterExpressionType() == ExpressionType.LESSTHAN
- || expressionTree.getFilterExpressionType() == ExpressionType.GREATERTHAN_EQUALTO
- || expressionTree.getFilterExpressionType() == ExpressionType.LESSTHAN_EQUALTO) {
-
- return new RowLevelRangeFilterResolverImpl(expression, isExpressionResolve, false,
- tableIdentifier);
- }
-
- return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false);
- }
- return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false);
- }
- break;
- default:
- if(expression instanceof ConditionalExpression) {
- condExpression = (ConditionalExpression) expression;
- if (condExpression.isSingleDimension()
- && condExpression.getColumnList().get(0).getCarbonColumn().getDataType()
- != DataType.ARRAY
- && condExpression.getColumnList().get(0).getCarbonColumn().getDataType()
- != DataType.STRUCT) {
- condExpression = (ConditionalExpression) expression;
- if (condExpression.getColumnList().get(0).getCarbonColumn()
- .hasEncoding(Encoding.DICTIONARY) && !condExpression.getColumnList().get(0)
- .getCarbonColumn().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
- return new ConditionalFilterResolverImpl(expression, true, true);
- }
- }
- }
- }
- return new RowLevelFilterResolverImpl(expression, false, false, tableIdentifier);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/FilterProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/FilterProcessor.java b/core/src/main/java/org/apache/carbondata/scan/filter/FilterProcessor.java
deleted file mode 100644
index 44161b2..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/FilterProcessor.java
+++ /dev/null
@@ -1,57 +0,0 @@
-package org.apache.carbondata.scan.filter;
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.datastore.DataRefNode;
-import org.apache.carbondata.core.datastore.block.AbstractIndex;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
-
-public interface FilterProcessor {
-
- /**
- * API will provide the resolved form of filters based on the filter
- * expression tree which is been passed.
- *
- * @param expressionTree , filter expression tree
- * @param tableIdentifier ,contains carbon store informations.
- * @return
- * @throws FilterUnsupportedException
- */
- FilterResolverIntf getFilterResolver(Expression expressionTree,
- AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException, IOException;
-
- /**
- * This API is exposed inorder to get the required block reference node
- * based on the filter.The block list will be send to the executer tasks inorder
- * to apply filters.
- *
- * @param filterResolver DataBlock list with resolved filters
- * @return list of DataRefNode.
- */
- List<DataRefNode> getFilterredBlocks(DataRefNode dataRefNode, FilterResolverIntf filterResolver,
- AbstractIndex segmentIndexBuilder, AbsoluteTableIdentifier tableIdentifier);
-
-}
[14/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeFilterExecuterImpl.java
deleted file mode 100644
index 6f9f49c..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeFilterExecuterImpl.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.executer;
-
-import java.io.IOException;
-import java.util.BitSet;
-
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
-import org.apache.carbondata.core.util.ByteUtil;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.scan.filter.FilterUtil;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.scan.processor.BlocksChunkHolder;
-
-public class IncludeFilterExecuterImpl implements FilterExecuter {
-
- protected DimColumnResolvedFilterInfo dimColumnEvaluatorInfo;
- protected DimColumnExecuterFilterInfo dimColumnExecuterInfo;
- protected SegmentProperties segmentProperties;
-
- public IncludeFilterExecuterImpl(DimColumnResolvedFilterInfo dimColumnEvaluatorInfo,
- SegmentProperties segmentProperties) {
- this.dimColumnEvaluatorInfo = dimColumnEvaluatorInfo;
- this.segmentProperties = segmentProperties;
- dimColumnExecuterInfo = new DimColumnExecuterFilterInfo();
- FilterUtil.prepareKeysFromSurrogates(dimColumnEvaluatorInfo.getFilterValues(),
- segmentProperties, dimColumnEvaluatorInfo.getDimension(), dimColumnExecuterInfo);
-
- }
-
- @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException {
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
- .get(dimColumnEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
- }
- return getFilteredIndexes(blockChunkHolder.getDimensionDataChunk()[blockIndex],
- blockChunkHolder.getDataBlock().nodeSize());
- }
-
- protected BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
- int numerOfRows) {
- if (dimensionColumnDataChunk.isNoDicitionaryColumn()
- && dimensionColumnDataChunk instanceof VariableLengthDimensionDataChunk) {
- return setDirectKeyFilterIndexToBitSet(
- (VariableLengthDimensionDataChunk) dimensionColumnDataChunk, numerOfRows);
- } else if (dimensionColumnDataChunk.isExplicitSorted()
- && dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
- return setFilterdIndexToBitSetWithColumnIndex(
- (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, numerOfRows);
- }
-
- return setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows);
- }
-
- private BitSet setDirectKeyFilterIndexToBitSet(
- VariableLengthDimensionDataChunk dimensionColumnDataChunk, int numerOfRows) {
- BitSet bitSet = new BitSet(numerOfRows);
- byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
- for (int i = 0; i < filterValues.length; i++) {
- byte[] filterVal = filterValues[i];
- if (dimensionColumnDataChunk.isExplicitSorted()) {
- for (int index = 0; index < numerOfRows; index++) {
- if (dimensionColumnDataChunk.compareTo(index, filterVal) == 0) {
- bitSet.set(dimensionColumnDataChunk.getInvertedIndex(index));
- }
- }
- } else {
- for (int index = 0; index < numerOfRows; index++) {
- if (dimensionColumnDataChunk.compareTo(index, filterVal) == 0) {
- bitSet.set(index);
- }
- }
- }
- }
- return bitSet;
-
- }
-
- private BitSet setFilterdIndexToBitSetWithColumnIndex(
- FixedLengthDimensionDataChunk dimensionColumnDataChunk, int numerOfRows) {
- BitSet bitSet = new BitSet(numerOfRows);
- int start = 0;
- int last = 0;
- int startIndex = 0;
- byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
- for (int i = 0; i < filterValues.length; i++) {
- start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
- filterValues[i], false);
- if (start < 0) {
- continue;
- }
- bitSet.set(dimensionColumnDataChunk.getInvertedIndex(start));
- last = start;
- for (int j = start + 1; j < numerOfRows; j++) {
- if (dimensionColumnDataChunk.compareTo(j, filterValues[i]) == 0) {
- bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j));
- last++;
- } else {
- break;
- }
- }
- startIndex = last;
- if (startIndex >= numerOfRows) {
- break;
- }
- }
- return bitSet;
- }
-
- private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
- int numerOfRows) {
- BitSet bitSet = new BitSet(numerOfRows);
- if (dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
- byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
- for (int k = 0; k < filterValues.length; k++) {
- for (int j = 0; j < numerOfRows; j++) {
- if (dimensionColumnDataChunk.compareTo(j, filterValues[k]) == 0) {
- bitSet.set(j);
- }
- }
- }
- }
- return bitSet;
- }
-
- public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) {
- BitSet bitSet = new BitSet(1);
- byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
- int columnIndex = dimColumnEvaluatorInfo.getColumnIndex();
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping().get(columnIndex);
-
- boolean isScanRequired = false;
- for (int k = 0; k < filterValues.length; k++) {
- // filter value should be in range of max and min value i.e
- // max>filtervalue>min
- // so filter-max should be negative
- int maxCompare =
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blkMaxVal[blockIndex]);
- // and filter-min should be positive
- int minCompare =
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blkMinVal[blockIndex]);
-
- // if any filter value is in range than this block needs to be
- // scanned
- if (maxCompare <= 0 && minCompare >= 0) {
- isScanRequired = true;
- break;
- }
- }
- if (isScanRequired) {
- bitSet.set(0);
- }
- return bitSet;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/executer/OrFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/OrFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/OrFilterExecuterImpl.java
deleted file mode 100644
index 1ca89b1..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/OrFilterExecuterImpl.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.executer;
-
-import java.io.IOException;
-import java.util.BitSet;
-
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.processor.BlocksChunkHolder;
-
-public class OrFilterExecuterImpl implements FilterExecuter {
-
- private FilterExecuter leftExecuter;
- private FilterExecuter rightExecuter;
-
- public OrFilterExecuterImpl(FilterExecuter leftExecuter, FilterExecuter rightExecuter) {
- this.leftExecuter = leftExecuter;
- this.rightExecuter = rightExecuter;
- }
-
- @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
- throws FilterUnsupportedException, IOException {
- BitSet leftFilters = leftExecuter.applyFilter(blockChunkHolder);
- BitSet rightFilters = rightExecuter.applyFilter(blockChunkHolder);
- leftFilters.or(rightFilters);
-
- return leftFilters;
- }
-
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
- BitSet leftFilters = leftExecuter.isScanRequired(blockMaxValue, blockMinValue);
- BitSet rightFilters = rightExecuter.isScanRequired(blockMaxValue, blockMinValue);
- leftFilters.or(rightFilters);
- return leftFilters;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/executer/RestructureFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RestructureFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/RestructureFilterExecuterImpl.java
deleted file mode 100644
index 26bd9b6..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RestructureFilterExecuterImpl.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.executer;
-
-import java.util.BitSet;
-
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.scan.filter.FilterUtil;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.scan.processor.BlocksChunkHolder;
-
-public class RestructureFilterExecuterImpl implements FilterExecuter {
-
- DimColumnExecuterFilterInfo dimColumnExecuterInfo;
-
- public RestructureFilterExecuterImpl(DimColumnResolvedFilterInfo dimColumnResolvedFilterInfo,
- SegmentProperties segmentProperties) {
- dimColumnExecuterInfo = new DimColumnExecuterFilterInfo();
- FilterUtil
- .prepareKeysFromSurrogates(dimColumnResolvedFilterInfo.getFilterValues(), segmentProperties,
- dimColumnResolvedFilterInfo.getDimension(), dimColumnExecuterInfo);
- }
-
- @Override public BitSet applyFilter(BlocksChunkHolder blocksChunkHolder) {
- BitSet bitSet = new BitSet(blocksChunkHolder.getDataBlock().nodeSize());
- byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
- if (null != filterValues && filterValues.length > 0) {
- bitSet.set(0, blocksChunkHolder.getDataBlock().nodeSize());
- }
- return bitSet;
- }
-
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
- BitSet bitSet = new BitSet(1);
- bitSet.set(0);
- return bitSet;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelFilterExecuterImpl.java
deleted file mode 100644
index b3d0c6e..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ /dev/null
@@ -1,368 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.executer;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.cache.dictionary.Dictionary;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.core.metadata.Encoding;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
-import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.scan.executor.util.QueryUtil;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.FilterUtil;
-import org.apache.carbondata.scan.filter.GenericQueryType;
-import org.apache.carbondata.scan.filter.intf.RowImpl;
-import org.apache.carbondata.scan.filter.intf.RowIntf;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.scan.processor.BlocksChunkHolder;
-
-public class RowLevelFilterExecuterImpl implements FilterExecuter {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(RowLevelFilterExecuterImpl.class.getName());
- protected List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList;
- protected List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList;
- protected Expression exp;
- protected AbsoluteTableIdentifier tableIdentifier;
- protected SegmentProperties segmentProperties;
- /**
- * it has index at which given dimension is stored in file
- */
- private int[] blocksIndex;
-
- private Map<Integer, GenericQueryType> complexDimensionInfoMap;
-
- public RowLevelFilterExecuterImpl(List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
- List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
- AbsoluteTableIdentifier tableIdentifier, SegmentProperties segmentProperties,
- Map<Integer, GenericQueryType> complexDimensionInfoMap) {
- this.dimColEvaluatorInfoList = dimColEvaluatorInfoList;
- this.segmentProperties = segmentProperties;
- this.blocksIndex = new int[dimColEvaluatorInfoList.size()];
- for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
- this.blocksIndex[i] = segmentProperties.getDimensionOrdinalToBlockMapping()
- .get(dimColEvaluatorInfoList.get(i).getColumnIndex());
- }
- if (null == msrColEvalutorInfoList) {
- this.msrColEvalutorInfoList = new ArrayList<MeasureColumnResolvedFilterInfo>(20);
- } else {
- this.msrColEvalutorInfoList = msrColEvalutorInfoList;
- }
- this.exp = exp;
- this.tableIdentifier = tableIdentifier;
- this.complexDimensionInfoMap = complexDimensionInfoMap;
- }
-
- @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
- throws FilterUnsupportedException, IOException {
- for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
- DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i);
- if (dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.ARRAY
- && dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.STRUCT) {
- if (null == blockChunkHolder.getDimensionDataChunk()[blocksIndex[i]]) {
- blockChunkHolder.getDimensionDataChunk()[blocksIndex[i]] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blocksIndex[i]);
- }
- } else {
- GenericQueryType complexType = complexDimensionInfoMap.get(blocksIndex[i]);
- complexType.fillRequiredBlockData(blockChunkHolder);
- }
- }
-
- // CHECKSTYLE:OFF Approval No:Approval-V1R2C10_001
- if (null != msrColEvalutorInfoList) {
- for (MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo : msrColEvalutorInfoList) {
- if (null == blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo
- .getColumnIndex()]) {
- blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo.getColumnIndex()] =
- blockChunkHolder.getDataBlock().getMeasureChunk(blockChunkHolder.getFileReader(),
- msrColumnEvalutorInfo.getColumnIndex());
- }
- }
- }
- // CHECKSTYLE:ON
-
- int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
- BitSet set = new BitSet(numberOfRows);
- RowIntf row = new RowImpl();
- boolean invalidRowsPresent = false;
- for (int index = 0; index < numberOfRows; index++) {
- createRow(blockChunkHolder, row, index);
- Boolean rslt = false;
- try {
- rslt = exp.evaluate(row).getBoolean();
- }
- // Any invalid member while evaluation shall be ignored, system will log the
- // error only once since all rows the evaluation happens so inorder to avoid
- // too much log inforation only once the log will be printed.
- catch (FilterIllegalMemberException e) {
- FilterUtil.logError(e, invalidRowsPresent);
- }
- if (null != rslt && rslt) {
- set.set(index);
- }
- }
- return set;
- }
-
- /**
- * Method will read the members of particular dimension block and create
- * a row instance for further processing of the filters
- *
- * @param blockChunkHolder
- * @param row
- * @param index
- * @throws IOException
- */
- private void createRow(BlocksChunkHolder blockChunkHolder, RowIntf row, int index)
- throws IOException {
- Object[] record = new Object[dimColEvaluatorInfoList.size() + msrColEvalutorInfoList.size()];
- String memberString;
- for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
- DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i);
- if (dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.ARRAY
- && dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.STRUCT) {
- if (!dimColumnEvaluatorInfo.isDimensionExistsInCurrentSilce()) {
- record[dimColumnEvaluatorInfo.getRowIndex()] = dimColumnEvaluatorInfo.getDefaultValue();
- }
- if (!dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)
- && blockChunkHolder
- .getDimensionDataChunk()[blocksIndex[i]] instanceof VariableLengthDimensionDataChunk) {
-
- VariableLengthDimensionDataChunk dimensionColumnDataChunk =
- (VariableLengthDimensionDataChunk) blockChunkHolder
- .getDimensionDataChunk()[blocksIndex[i]];
- memberString = readMemberBasedOnNoDictionaryVal(dimensionColumnDataChunk, index);
- if (null != memberString) {
- if (memberString.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
- memberString = null;
- }
- record[dimColumnEvaluatorInfo.getRowIndex()] = DataTypeUtil
- .getDataBasedOnDataType(memberString,
- dimColumnEvaluatorInfo.getDimension().getDataType());
- } else {
- continue;
- }
- } else {
- int dictionaryValue =
- readSurrogatesFromColumnBlock(blockChunkHolder, index, dimColumnEvaluatorInfo,
- blocksIndex[i]);
- if (dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)
- && !dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
- memberString =
- getFilterActualValueFromDictionaryValue(dimColumnEvaluatorInfo, dictionaryValue);
- record[dimColumnEvaluatorInfo.getRowIndex()] = DataTypeUtil
- .getDataBasedOnDataType(memberString,
- dimColumnEvaluatorInfo.getDimension().getDataType());
- } else if (dimColumnEvaluatorInfo.getDimension()
- .hasEncoding(Encoding.DIRECT_DICTIONARY)) {
-
- Object member = getFilterActualValueFromDirectDictionaryValue(dimColumnEvaluatorInfo,
- dictionaryValue);
- record[dimColumnEvaluatorInfo.getRowIndex()] = member;
- }
- }
- } else {
- try {
- GenericQueryType complexType = complexDimensionInfoMap.get(blocksIndex[i]);
- ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
- DataOutputStream dataOutputStream = new DataOutputStream(byteStream);
- complexType
- .parseBlocksAndReturnComplexColumnByteArray(blockChunkHolder.getDimensionDataChunk(),
- index, dataOutputStream);
- record[dimColumnEvaluatorInfo.getRowIndex()] = complexType
- .getDataBasedOnDataTypeFromSurrogates(ByteBuffer.wrap(byteStream.toByteArray()));
- byteStream.close();
- } catch (IOException e) {
- LOGGER.info(e.getMessage());
- }
- }
- }
-
- DataType msrType;
-
- for (MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo : msrColEvalutorInfoList) {
- switch (msrColumnEvalutorInfo.getType()) {
- case INT:
- case LONG:
- msrType = DataType.LONG;
- break;
- case DECIMAL:
- msrType = DataType.DECIMAL;
- break;
- default:
- msrType = DataType.DOUBLE;
- }
- // if measure doesnt exist then set the default value.
- Object msrValue;
- switch (msrType) {
- case INT:
- case LONG:
- msrValue = blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo.getColumnIndex()]
- .getMeasureDataHolder().getReadableLongValueByIndex(index);
- break;
- case DECIMAL:
- msrValue = blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo.getColumnIndex()]
- .getMeasureDataHolder().getReadableBigDecimalValueByIndex(index);
- break;
- default:
- msrValue = blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo.getColumnIndex()]
- .getMeasureDataHolder().getReadableDoubleValueByIndex(index);
- }
- record[msrColumnEvalutorInfo.getRowIndex()] =
- blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo.getColumnIndex()]
- .getNullValueIndexHolder().getBitSet().get(index) ? null : msrValue;
- }
- row.setValues(record);
- }
-
- /**
- * method will read the actual data from the direct dictionary generator
- * by passing direct dictionary value.
- *
- * @param dimColumnEvaluatorInfo
- * @param dictionaryValue
- * @return
- */
- private Object getFilterActualValueFromDirectDictionaryValue(
- DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, int dictionaryValue) {
- Object memberString = null;
- DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
- .getDirectDictionaryGenerator(dimColumnEvaluatorInfo.getDimension().getDataType());
- if (null != directDictionaryGenerator) {
- memberString = directDictionaryGenerator.getValueFromSurrogate(dictionaryValue);
- }
- return memberString;
- }
-
- /**
- * Read the actual filter member by passing the dictionary value from
- * the forward dictionary cache which which holds column wise cache
- *
- * @param dimColumnEvaluatorInfo
- * @param dictionaryValue
- * @return
- * @throws IOException
- */
- private String getFilterActualValueFromDictionaryValue(
- DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, int dictionaryValue) throws IOException {
- String memberString;
- Dictionary forwardDictionary = FilterUtil
- .getForwardDictionaryCache(tableIdentifier, dimColumnEvaluatorInfo.getDimension());
-
- memberString = forwardDictionary.getDictionaryValueForKey(dictionaryValue);
- if (null != memberString) {
- if (memberString.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
- memberString = null;
- }
- }
- return memberString;
- }
-
- /**
- * read the filter member dictionary data from the block corresponding to
- * applied filter column
- *
- * @param blockChunkHolder
- * @param index
- * @param dimColumnEvaluatorInfo
- * @return
- */
- private int readSurrogatesFromColumnBlock(BlocksChunkHolder blockChunkHolder, int index,
- DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, int blockIndex) {
- if (dimColumnEvaluatorInfo.getDimension().isColumnar()) {
- byte[] rawData = blockChunkHolder.getDimensionDataChunk()[blockIndex].getChunkData(index);
- ByteBuffer byteBuffer = ByteBuffer.allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE);
- int dictionaryValue = CarbonUtil.getSurrogateKey(rawData, byteBuffer);
- return dictionaryValue;
- } else {
- return readSurrogatesFromColumnGroupBlock(blockChunkHolder, index, dimColumnEvaluatorInfo,
- blockIndex);
- }
-
- }
-
- /**
- * @param blockChunkHolder
- * @param index
- * @param dimColumnEvaluatorInfo
- * @return read surrogate of given row of given column group dimension
- */
- private int readSurrogatesFromColumnGroupBlock(BlocksChunkHolder blockChunkHolder, int index,
- DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, int blockIndex) {
- try {
- KeyStructureInfo keyStructureInfo =
- QueryUtil.getKeyStructureInfo(segmentProperties, dimColumnEvaluatorInfo);
- byte[] colData = blockChunkHolder.getDimensionDataChunk()[blockIndex].getChunkData(index);
- long[] result = keyStructureInfo.getKeyGenerator().getKeyArray(colData);
- int colGroupId =
- QueryUtil.getColumnGroupId(segmentProperties, dimColumnEvaluatorInfo.getColumnIndex());
- int dictionaryValue = (int) result[segmentProperties
- .getColumnGroupMdKeyOrdinal(colGroupId, dimColumnEvaluatorInfo.getColumnIndex())];
- return dictionaryValue;
- } catch (KeyGenException e) {
- LOGGER.error(e);
- }
- return 0;
- }
-
- /**
- * Reading the blocks for no dictionary data, in no dictionary case
- * directly the filter data will read, no need to scan the dictionary
- * or read the dictionary value.
- *
- * @param dimensionColumnDataChunk
- * @param index
- * @return
- */
- private String readMemberBasedOnNoDictionaryVal(
- VariableLengthDimensionDataChunk dimensionColumnDataChunk, int index) {
- return new String(dimensionColumnDataChunk.getChunkData(index),
- Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
- }
-
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
- BitSet bitSet = new BitSet(1);
- bitSet.set(0);
- return bitSet;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
deleted file mode 100644
index 935a118..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.executer;
-
-import java.io.IOException;
-import java.util.BitSet;
-import java.util.List;
-
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
-import org.apache.carbondata.core.metadata.Encoding;
-import org.apache.carbondata.core.util.ByteUtil;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.scan.processor.BlocksChunkHolder;
-
-public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecuterImpl {
- private byte[][] filterRangeValues;
-
- public RowLevelRangeGrtThanFiterExecuterImpl(
- List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
- List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
- AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues,
- SegmentProperties segmentProperties) {
- super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties,
- null);
- this.filterRangeValues = filterRangeValues;
- }
-
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
- BitSet bitSet = new BitSet(1);
- byte[][] filterValues = this.filterRangeValues;
- int columnIndex = this.dimColEvaluatorInfoList.get(0).getColumnIndex();
- boolean isScanRequired = false;
- for (int k = 0; k < filterValues.length; k++) {
- // filter value should be in range of max and min value i.e
- // max>filtervalue>min
- // so filter-max should be negative
- int maxCompare =
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue[columnIndex]);
- // if any filter value is in range than this block needs to be
- // scanned means always less than block max range.
- if (maxCompare < 0) {
- isScanRequired = true;
- break;
- }
- }
- if (isScanRequired) {
- bitSet.set(0);
- }
- return bitSet;
-
- }
-
- @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
- throws FilterUnsupportedException, IOException {
- if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
- return super.applyFilter(blockChunkHolder);
- }
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
- .get(dimColEvaluatorInfoList.get(0).getColumnIndex());
- if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
- }
- return getFilteredIndexes(blockChunkHolder.getDimensionDataChunk()[blockIndex],
- blockChunkHolder.getDataBlock().nodeSize());
- }
-
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
- int numerOfRows) {
- if (dimensionColumnDataChunk.isExplicitSorted()
- && dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
- return setFilterdIndexToBitSetWithColumnIndex(
- (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, numerOfRows);
- }
- return setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows);
- }
-
- /**
- * Method will scan the block and finds the range start index from which all members
- * will be considered for applying range filters. this method will be called if the
- * column is not supported by default so column index mapping will be present for
- * accesing the members from the block.
- *
- * @param dimensionColumnDataChunk
- * @param numerOfRows
- * @return BitSet.
- */
- private BitSet setFilterdIndexToBitSetWithColumnIndex(
- FixedLengthDimensionDataChunk dimensionColumnDataChunk, int numerOfRows) {
- BitSet bitSet = new BitSet(numerOfRows);
- int start = 0;
- int last = 0;
- int startIndex = 0;
- byte[][] filterValues = this.filterRangeValues;
- for (int i = 0; i < filterValues.length; i++) {
- start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
- filterValues[i], true);
- if (start >= 0) {
- start = CarbonUtil.nextGreaterValueToTarget(start, dimensionColumnDataChunk,
- filterValues[i], numerOfRows);
- }
- // Logic will handle the case where the range filter member is not present in block
- // in this case the binary search will return the index from where the bit sets will be
- // set inorder to apply filters. this is greater than filter so the range will be taken
- // from the next element which is greater than filter member.
- if (start < 0) {
- start = -(start + 1);
- if (start == numerOfRows) {
- start = start - 1;
- }
- // Method will compare the tentative index value after binary search, this tentative
- // index needs to be compared by the filter member if its > filter then from that
- // index the bitset will be considered for filtering process.
- if (ByteUtil.compare(filterValues[i],
- dimensionColumnDataChunk.getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))
- > 0) {
- start = start + 1;
- }
- }
-
- last = start;
- for (int j = start; j < numerOfRows; j++) {
- bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j));
- last++;
- }
- startIndex = last;
- if (startIndex >= numerOfRows) {
- break;
- }
- }
-
- return bitSet;
- }
-
- /**
- * Method will scan the block and finds the range start index from which all
- * members will be considered for applying range filters. this method will
- * be called if the column is sorted default so column index
- * mapping will be present for accesing the members from the block.
- *
- * @param dimensionColumnDataChunk
- * @param numerOfRows
- * @return BitSet.
- */
- private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
- int numerOfRows) {
- BitSet bitSet = new BitSet(numerOfRows);
- if (dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
- int start = 0;
- int last = 0;
- int startIndex = 0;
- byte[][] filterValues = this.filterRangeValues;
- for (int k = 0; k < filterValues.length; k++) {
- start = CarbonUtil.getFirstIndexUsingBinarySearch(
- (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, startIndex, numerOfRows - 1,
- filterValues[k], true);
- if(start >= 0){
- start = CarbonUtil.nextGreaterValueToTarget(start,
- (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, filterValues[k],
- numerOfRows);
- }
- if (start < 0) {
- start = -(start + 1);
- if (start == numerOfRows) {
- start = start - 1;
- }
- // Method will compare the tentative index value after binary search, this tentative
- // index needs to be compared by the filter member if its > filter then from that
- // index the bitset will be considered for filtering process.
- if (ByteUtil.compare(filterValues[k], dimensionColumnDataChunk.getChunkData(start)) > 0) {
- start = start + 1;
- }
- }
- last = start;
- for (int j = start; j < numerOfRows; j++) {
- bitSet.set(j);
- last++;
- }
- startIndex = last;
- if (startIndex >= numerOfRows) {
- break;
- }
- }
- }
- return bitSet;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
deleted file mode 100644
index df96b7a..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.executer;
-
-import java.io.IOException;
-import java.util.BitSet;
-import java.util.List;
-
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
-import org.apache.carbondata.core.metadata.Encoding;
-import org.apache.carbondata.core.util.ByteUtil;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.scan.processor.BlocksChunkHolder;
-
-public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilterExecuterImpl {
-
- protected byte[][] filterRangeValues;
-
- public RowLevelRangeGrtrThanEquaToFilterExecuterImpl(
- List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
- List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
- AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues,
- SegmentProperties segmentProperties) {
- super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties,
- null);
- this.filterRangeValues = filterRangeValues;
- }
-
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
- BitSet bitSet = new BitSet(1);
- byte[][] filterValues = this.filterRangeValues;
- int columnIndex = this.dimColEvaluatorInfoList.get(0).getColumnIndex();
- boolean isScanRequired = false;
- for (int k = 0; k < filterValues.length; k++) {
- // filter value should be in range of max and min value i.e
- // max>filtervalue>min
- // so filter-max should be negative
- int maxCompare =
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue[columnIndex]);
- // if any filter value is in range than this block needs to be
- // scanned less than equal to max range.
- if (maxCompare <= 0) {
- isScanRequired = true;
- break;
- }
- }
- if (isScanRequired) {
- bitSet.set(0);
- }
- return bitSet;
-
- }
-
- @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
- throws FilterUnsupportedException, IOException {
- if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
- return super.applyFilter(blockChunkHolder);
- }
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
- .get(dimColEvaluatorInfoList.get(0).getColumnIndex());
- if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
- }
- return getFilteredIndexes(blockChunkHolder.getDimensionDataChunk()[blockIndex],
- blockChunkHolder.getDataBlock().nodeSize());
- }
-
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
- int numerOfRows) {
- if (dimensionColumnDataChunk.isExplicitSorted()
- && dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
- return setFilterdIndexToBitSetWithColumnIndex(
- (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, numerOfRows);
- }
- return setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows);
- }
-
- /**
- * Method will scan the block and finds the range start index from which all members
- * will be considered for applying range filters. this method will be called if the
- * column is not supported by default so column index mapping will be present for
- * accesing the members from the block.
- *
- * @param dimensionColumnDataChunk
- * @param numerOfRows
- * @return BitSet.
- */
- private BitSet setFilterdIndexToBitSetWithColumnIndex(
- FixedLengthDimensionDataChunk dimensionColumnDataChunk, int numerOfRows) {
- BitSet bitSet = new BitSet(numerOfRows);
- int start = 0;
- int last = 0;
- int startIndex = 0;
- byte[][] filterValues = this.filterRangeValues;
- for (int i = 0; i < filterValues.length; i++) {
- start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
- filterValues[i], false);
- if (start < 0) {
- start = -(start + 1);
- if (start == numerOfRows) {
- start = start - 1;
- }
- // Method will compare the tentative index value after binary search, this tentative
- // index needs to be compared by the filter member if its >= filter then from that
- // index the bitset will be considered for filtering process.
- if (ByteUtil.compare(filterValues[i],
- dimensionColumnDataChunk.getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))
- >= 0) {
- start = start + 1;
- }
- }
- last = start;
- for (int j = start; j < numerOfRows; j++) {
- bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j));
- last++;
- }
- startIndex = last;
- if (startIndex >= numerOfRows) {
- break;
- }
- }
- return bitSet;
- }
-
- /**
- * Method will scan the block and finds the range start index from which all
- * members will be considered for applying range filters. this method will
- * be called if the column is sorted default so column index
- * mapping will be present for accesing the members from the block.
- *
- * @param dimensionColumnDataChunk
- * @param numerOfRows
- * @return BitSet.
- */
- private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
- int numerOfRows) {
- BitSet bitSet = new BitSet(numerOfRows);
- if (dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
- int start = 0;
- int last = 0;
- int startIndex = 0;
- byte[][] filterValues = this.filterRangeValues;
- for (int k = 0; k < filterValues.length; k++) {
- start = CarbonUtil.getFirstIndexUsingBinarySearch(
- (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, startIndex, numerOfRows - 1,
- filterValues[k], false);
- if (start < 0) {
- start = -(start + 1);
- if (start == numerOfRows) {
- start = start - 1;
- }
- // Method will compare the tentative index value after binary search, this tentative
- // index needs to be compared by the filter member if its >= filter then from that
- // index the bitset will be considered for filtering process.
- if (ByteUtil.compare(filterValues[k], dimensionColumnDataChunk.getChunkData(start))
- >= 0) {
- start = start + 1;
- }
- }
-
- last = start;
- for (int j = start; j < numerOfRows; j++) {
- bitSet.set(j);
- last++;
- }
- startIndex = last;
- if (startIndex >= numerOfRows) {
- break;
- }
- }
- }
- return bitSet;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
deleted file mode 100644
index 95fb3aa..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ /dev/null
@@ -1,248 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.executer;
-
-import java.io.IOException;
-import java.util.BitSet;
-import java.util.List;
-
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
-import org.apache.carbondata.core.metadata.Encoding;
-import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
-import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
-import org.apache.carbondata.core.util.ByteUtil;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.FilterUtil;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.scan.processor.BlocksChunkHolder;
-
-public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilterExecuterImpl {
- protected byte[][] filterRangeValues;
-
- public RowLevelRangeLessThanEqualFilterExecuterImpl(
- List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
- List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
- AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues,
- SegmentProperties segmentProperties) {
- super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties,
- null);
- this.filterRangeValues = filterRangeValues;
- }
-
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
- BitSet bitSet = new BitSet(1);
- byte[][] filterValues = this.filterRangeValues;
- int columnIndex = this.dimColEvaluatorInfoList.get(0).getColumnIndex();
- boolean isScanRequired = false;
- for (int k = 0; k < filterValues.length; k++) {
- // and filter-min should be positive
- int minCompare =
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMinValue[columnIndex]);
-
- // if any filter applied is not in range of min and max of block
- // then since its a less than equal to fiter validate whether the block
- // min range is less than equal to applied filter member
- if (minCompare >= 0) {
- isScanRequired = true;
- break;
- }
- }
- if (isScanRequired) {
- bitSet.set(0);
- }
- return bitSet;
-
- }
-
- @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
- throws FilterUnsupportedException, IOException {
- if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
- return super.applyFilter(blockChunkHolder);
- }
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
- .get(dimColEvaluatorInfoList.get(0).getColumnIndex());
- if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
- }
- return getFilteredIndexes(blockChunkHolder.getDimensionDataChunk()[blockIndex],
- blockChunkHolder.getDataBlock().nodeSize());
- }
-
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
- int numerOfRows) {
- byte[] defaultValue = null;
- if (dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
- DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
- .getDirectDictionaryGenerator(
- dimColEvaluatorInfoList.get(0).getDimension().getDataType());
- int key = directDictionaryGenerator.generateDirectSurrogateKey(null) + 1;
- defaultValue = FilterUtil.getMaskKey(key, dimColEvaluatorInfoList.get(0).getDimension(),
- this.segmentProperties.getDimensionKeyGenerator());
- }
- if (dimensionColumnDataChunk.isExplicitSorted()
- && dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
-
- return setFilterdIndexToBitSetWithColumnIndex(
- (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, numerOfRows, defaultValue);
-
- }
- return setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows, defaultValue);
- }
-
- /**
- * Method will scan the block and finds the range start index from which all members
- * will be considered for applying range filters. this method will be called if the
- * column is not supported by default so column index mapping will be present for
- * accesing the members from the block.
- *
- * @param dimensionColumnDataChunk
- * @param numerOfRows
- * @return BitSet.
- */
- private BitSet setFilterdIndexToBitSetWithColumnIndex(
- FixedLengthDimensionDataChunk dimensionColumnDataChunk, int numerOfRows,
- byte[] defaultValue) {
- BitSet bitSet = new BitSet(numerOfRows);
- int start = 0;
- int last = 0;
- int skip = 0;
- int startIndex = 0;
- byte[][] filterValues = this.filterRangeValues;
- //find the number of default values to skip the null value in case of direct dictionary
- if (null != defaultValue) {
- start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
- defaultValue, true);
- if (start < 0) {
- skip = -(start + 1);
- // end of block
- if (skip == numerOfRows) {
- return bitSet;
- }
- } else {
- skip = start;
- }
- startIndex = skip;
- }
- for (int i = 0; i < filterValues.length; i++) {
- start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
- filterValues[i], true);
- if (start < 0) {
- start = -(start + 1);
- if (start == numerOfRows) {
- start = start - 1;
- }
- // Method will compare the tentative index value after binary search, this tentative
- // index needs to be compared by the filter member if its >= filter then from that
- // index the bitset will be considered for filtering process.
- if (ByteUtil.compare(filterValues[i],
- dimensionColumnDataChunk.getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))
- <= 0) {
- start = start - 1;
- }
- }
- last = start;
- for (int j = start; j >= skip; j--) {
- bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j));
- last--;
- }
- startIndex = last;
- if (startIndex <= 0) {
- break;
- }
- }
- return bitSet;
- }
-
- /**
- * Method will scan the block and finds the range start index from which all
- * members will be considered for applying range filters. this method will
- * be called if the column is sorted default so column index
- * mapping will be present for accesing the members from the block.
- *
- * @param dimensionColumnDataChunk
- * @param numerOfRows
- * @param defaultValue
- * @return BitSet.
- */
- private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
- int numerOfRows, byte[] defaultValue) {
- BitSet bitSet = new BitSet(numerOfRows);
- if (dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
- int start = 0;
- int last = 0;
- int startIndex = 0;
- byte[][] filterValues = this.filterRangeValues;
- int skip = 0;
- //find the number of default values to skip the null value in case of direct dictionary
- if (null != defaultValue) {
- start = CarbonUtil.getFirstIndexUsingBinarySearch(
- (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, startIndex, numerOfRows - 1,
- defaultValue, true);
- if (start < 0) {
- skip = -(start + 1);
- // end of block
- if (skip == numerOfRows) {
- return bitSet;
- }
- } else {
- skip = start;
- }
- startIndex = skip;
- }
- for (int k = 0; k < filterValues.length; k++) {
- start = CarbonUtil.getFirstIndexUsingBinarySearch(
- (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, startIndex, numerOfRows - 1,
- filterValues[k], true);
- if (start < 0) {
- start = -(start + 1);
- if (start == numerOfRows) {
- start = start - 1;
- }
- // Method will compare the tentative index value after binary search, this tentative
- // index needs to be compared by the filter member if its <= filter then from that
- // index the bitset will be considered for filtering process.
- if (ByteUtil.compare(filterValues[k], dimensionColumnDataChunk.getChunkData(start))
- <= 0) {
- start = start - 1;
- }
- }
- last = start;
- for (int j = start; j >= skip; j--) {
- bitSet.set(j);
- last--;
- }
- startIndex = last;
- if (startIndex <= 0) {
- break;
- }
- }
- }
- return bitSet;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
deleted file mode 100644
index 6233ce8..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
+++ /dev/null
@@ -1,255 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.executer;
-
-import java.io.IOException;
-import java.util.BitSet;
-import java.util.List;
-
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
-import org.apache.carbondata.core.metadata.Encoding;
-import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
-import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
-import org.apache.carbondata.core.util.ByteUtil;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.FilterUtil;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-import org.apache.carbondata.scan.processor.BlocksChunkHolder;
-
-public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecuterImpl {
- private byte[][] filterRangeValues;
-
- public RowLevelRangeLessThanFiterExecuterImpl(
- List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
- List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
- AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues,
- SegmentProperties segmentProperties) {
- super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties,
- null);
- this.filterRangeValues = filterRangeValues;
- }
-
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
- BitSet bitSet = new BitSet(1);
- byte[][] filterValues = this.filterRangeValues;
- int columnIndex = this.dimColEvaluatorInfoList.get(0).getColumnIndex();
- boolean isScanRequired = false;
- for (int k = 0; k < filterValues.length; k++) {
- // and filter-min should be positive
- int minCompare =
- ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMinValue[columnIndex]);
-
- // if any filter applied is not in range of min and max of block
- // then since its a less than fiter validate whether the block
- // min range is less than applied filter member
- if (minCompare > 0) {
- isScanRequired = true;
- break;
- }
- }
- if (isScanRequired) {
- bitSet.set(0);
- }
- return bitSet;
-
- }
-
- @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
- throws FilterUnsupportedException, IOException {
- if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
- return super.applyFilter(blockChunkHolder);
- }
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
- .get(dimColEvaluatorInfoList.get(0).getColumnIndex());
- if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
- }
- return getFilteredIndexes(blockChunkHolder.getDimensionDataChunk()[blockIndex],
- blockChunkHolder.getDataBlock().nodeSize());
- }
-
- private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
- int numerOfRows) {
- byte[] defaultValue = null;
- if (dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
- DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
- .getDirectDictionaryGenerator(
- dimColEvaluatorInfoList.get(0).getDimension().getDataType());
- int key = directDictionaryGenerator.generateDirectSurrogateKey(null) + 1;
- defaultValue = FilterUtil.getMaskKey(key, dimColEvaluatorInfoList.get(0).getDimension(),
- this.segmentProperties.getDimensionKeyGenerator());
- }
- if (dimensionColumnDataChunk.isExplicitSorted()
- && dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
- return setFilterdIndexToBitSetWithColumnIndex(
- (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, numerOfRows, defaultValue);
- }
- return setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows, defaultValue);
- }
-
- /**
- * Method will scan the block and finds the range start index from which all members
- * will be considered for applying range filters. this method will be called if the
- * column is not supported by default so column index mapping will be present for
- * accesing the members from the block.
- *
- * @param dimensionColumnDataChunk
- * @param numerOfRows
- * @return BitSet.
- */
- private BitSet setFilterdIndexToBitSetWithColumnIndex(
- FixedLengthDimensionDataChunk dimensionColumnDataChunk, int numerOfRows,
- byte[] defaultValue) {
- BitSet bitSet = new BitSet(numerOfRows);
- int start = 0;
- int last = 0;
- int startIndex = 0;
- int skip = 0;
- byte[][] filterValues = this.filterRangeValues;
-
- //find the number of default values to skip the null value in case of direct dictionary
- if (null != defaultValue) {
- start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
- defaultValue, false);
- if (start < 0) {
- skip = -(start + 1);
- // end of block
- if (skip == numerOfRows) {
- return bitSet;
- }
- } else {
- skip = start;
- }
- startIndex = skip;
- }
-
- for (int i = 0; i < filterValues.length; i++) {
- start = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
- filterValues[i], false);
- // Logic will handle the case where the range filter member is not present in block
- // in this case the binary search will return the index from where the bit sets will be
- // set inorder to apply filters. this is Lesser than filter so the range will be taken
- // from the prev element which is Lesser than filter member.
- start = CarbonUtil.nextLesserValueToTarget(start, dimensionColumnDataChunk, filterValues[i]);
- if (start < 0) {
- start = -(start + 1);
- if (start == numerOfRows) {
- start = start - 1;
- }
- // Method will compare the tentative index value after binary search, this tentative
- // index needs to be compared by the filter member if its < filter then from that
- // index the bitset will be considered for filtering process.
- if (ByteUtil.compare(filterValues[i],
- dimensionColumnDataChunk.getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))
- < 0) {
- start = start - 1;
- }
- }
- last = start;
- for (int j = start; j >= skip; j--) {
- bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j));
- last--;
- }
- startIndex = last;
- if (startIndex >= 0) {
- break;
- }
- }
- return bitSet;
- }
-
- /**
- * Method will scan the block and finds the range start index from which all
- * members will be considered for applying range filters. this method will
- * be called if the column is sorted default so column index
- * mapping will be present for accesing the members from the block.
- *
- * @param dimensionColumnDataChunk
- * @param numerOfRows
- * @return BitSet.
- */
- private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
- int numerOfRows, byte[] defaultValue) {
- BitSet bitSet = new BitSet(numerOfRows);
- if (dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
- int start = 0;
- int last = 0;
- int startIndex = 0;
- int skip = 0;
- byte[][] filterValues = this.filterRangeValues;
- //find the number of default values to skip the null value in case of direct dictionary
- if (null != defaultValue) {
- start = CarbonUtil.getFirstIndexUsingBinarySearch(
- (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, startIndex, numerOfRows - 1,
- defaultValue, false);
- if (start < 0) {
- skip = -(start + 1);
- // end of block
- if (skip == numerOfRows) {
- return bitSet;
- }
- } else {
- skip = start;
- }
- startIndex = skip;
- }
- for (int k = 0; k < filterValues.length; k++) {
- start = CarbonUtil.getFirstIndexUsingBinarySearch(
- (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, startIndex, numerOfRows - 1,
- filterValues[k], false);
- if(start >= 0) {
- start = CarbonUtil.nextLesserValueToTarget(start,
- (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, filterValues[k]);
- }
- if (start < 0) {
- start = -(start + 1);
-
- if (start >= numerOfRows) {
- start = numerOfRows - 1;
- }
- // Method will compare the tentative index value after binary search, this tentative
- // index needs to be compared by the filter member if its < filter then from that
- // index the bitset will be considered for filtering process.
- if (ByteUtil.compare(filterValues[k], dimensionColumnDataChunk.getChunkData(start)) < 0) {
- start = start - 1;
- }
- }
- last = start;
- for (int j = start; j >= skip; j--) {
- bitSet.set(j);
- last--;
- }
- startIndex = last;
- if (startIndex <= 0) {
- break;
- }
- }
- }
- return bitSet;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java
deleted file mode 100644
index 1bb42c7..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.executer;
-
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.scan.filter.intf.FilterExecuterType;
-import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
-import org.apache.carbondata.scan.filter.resolver.RowLevelRangeFilterResolverImpl;
-
-public class RowLevelRangeTypeExecuterFacory {
-
- private RowLevelRangeTypeExecuterFacory() {
-
- }
-
- /**
- * The method returns the Row Level Range fiter type instance based on
- * filter tree resolver type.
- *
- * @param filterExpressionResolverTree
- * @param segmentProperties
- * @return the generator instance
- */
- public static RowLevelFilterExecuterImpl getRowLevelRangeTypeExecuter(
- FilterExecuterType filterExecuterType, FilterResolverIntf filterExpressionResolverTree,
- SegmentProperties segmentProperties) {
- switch (filterExecuterType) {
-
- case ROWLEVEL_LESSTHAN:
- return new RowLevelRangeLessThanFiterExecuterImpl(
- ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
- .getDimColEvaluatorInfoList(),
- ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
- .getMsrColEvalutorInfoList(),
- filterExpressionResolverTree.getFilterExpression(),
- ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(),
- ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
- .getFilterRangeValues(segmentProperties), segmentProperties);
- case ROWLEVEL_LESSTHAN_EQUALTO:
- return new RowLevelRangeLessThanEqualFilterExecuterImpl(
- ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
- .getDimColEvaluatorInfoList(),
- ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
- .getMsrColEvalutorInfoList(),
- filterExpressionResolverTree.getFilterExpression(),
- ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(),
- ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
- .getFilterRangeValues(segmentProperties), segmentProperties);
- case ROWLEVEL_GREATERTHAN_EQUALTO:
- return new RowLevelRangeGrtrThanEquaToFilterExecuterImpl(
- ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
- .getDimColEvaluatorInfoList(),
- ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
- .getMsrColEvalutorInfoList(),
- filterExpressionResolverTree.getFilterExpression(),
- ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(),
- ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
- .getFilterRangeValues(segmentProperties), segmentProperties);
- case ROWLEVEL_GREATERTHAN:
- return new RowLevelRangeGrtThanFiterExecuterImpl(
- ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
- .getDimColEvaluatorInfoList(),
- ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
- .getMsrColEvalutorInfoList(),
- filterExpressionResolverTree.getFilterExpression(),
- ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(),
- ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
- .getFilterRangeValues(segmentProperties), segmentProperties);
- default:
- // Scenario wont come logic must break
- return null;
-
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/intf/ExpressionType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/intf/ExpressionType.java b/core/src/main/java/org/apache/carbondata/scan/filter/intf/ExpressionType.java
deleted file mode 100644
index 0dca5a4..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/intf/ExpressionType.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.filter.intf;
-
-public enum ExpressionType {
-
- AND,
- OR,
- NOT,
- EQUALS,
- NOT_EQUALS,
- LESSTHAN,
- LESSTHAN_EQUALTO,
- GREATERTHAN,
- GREATERTHAN_EQUALTO,
- ADD,
- SUBSTRACT,
- DIVIDE,
- MULTIPLY,
- IN,
- LIST,
- NOT_IN,
- UNKNOWN,
- LITERAL,
- RANGE,
- FALSE
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/intf/FilterExecuterType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/intf/FilterExecuterType.java b/core/src/main/java/org/apache/carbondata/scan/filter/intf/FilterExecuterType.java
deleted file mode 100644
index 39b7979..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/intf/FilterExecuterType.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.intf;
-
-import java.io.Serializable;
-
-public enum FilterExecuterType implements Serializable {
-
- INCLUDE, EXCLUDE, OR, AND, RESTRUCTURE, ROWLEVEL, RANGE, ROWLEVEL_GREATERTHAN,
- ROWLEVEL_GREATERTHAN_EQUALTO, ROWLEVEL_LESSTHAN_EQUALTO, ROWLEVEL_LESSTHAN
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/intf/RowImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/intf/RowImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/intf/RowImpl.java
deleted file mode 100644
index e924e79..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/intf/RowImpl.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.filter.intf;
-
-public class RowImpl implements RowIntf {
- private Object[] row;
-
- public RowImpl() {
- row = new Object[0];
- }
-
- @Override public Object getVal(int index) {
- return row[index];
- }
-
- @Override public Object[] getValues() {
- return row;
- }
-
- @Override public void setValues(final Object[] row) {
- this.row = row;
- }
-
- @Override public int size() {
- return this.row.length;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/intf/RowIntf.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/intf/RowIntf.java b/core/src/main/java/org/apache/carbondata/scan/filter/intf/RowIntf.java
deleted file mode 100644
index d4b2a8f..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/intf/RowIntf.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.filter.intf;
-
-public interface RowIntf {
- Object getVal(int index);
-
- Object[] getValues();
-
- void setValues(Object[] setValues);
-
- int size();
-
-}
[23/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
new file mode 100644
index 0000000..47e9f09
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter.executer;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.keygenerator.KeyGenerator;
+import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.executor.util.QueryUtil;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+
+/**
+ * It checks if filter is required on given block and if required, it does
+ * linear search on block data and set the bitset.
+ */
+public class ExcludeColGroupFilterExecuterImpl extends ExcludeFilterExecuterImpl {
+
+ /**
+ * LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(ExcludeColGroupFilterExecuterImpl.class.getName());
+
+ /**
+ * @param dimColResolvedFilterInfo
+ * @param segmentProperties
+ */
+ public ExcludeColGroupFilterExecuterImpl(DimColumnResolvedFilterInfo dimColResolvedFilterInfo,
+ SegmentProperties segmentProperties) {
+ super(dimColResolvedFilterInfo, segmentProperties);
+ }
+
+ /**
+ * It fills BitSet with row index which matches filter key
+ */
+ protected BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ int numerOfRows) {
+ BitSet bitSet = new BitSet(numerOfRows);
+ bitSet.flip(0, numerOfRows);
+ try {
+ KeyStructureInfo keyStructureInfo = getKeyStructureInfo();
+ byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
+ for (int i = 0; i < filterValues.length; i++) {
+ byte[] filterVal = filterValues[i];
+ for (int rowId = 0; rowId < numerOfRows; rowId++) {
+ byte[] colData = new byte[keyStructureInfo.getMaskByteRanges().length];
+ dimensionColumnDataChunk.fillChunkData(colData, 0, rowId, keyStructureInfo);
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterVal, colData) == 0) {
+ bitSet.flip(rowId);
+ }
+ }
+ }
+
+ } catch (Exception e) {
+ LOGGER.error(e);
+ }
+
+ return bitSet;
+ }
+
+ /**
+ * It is required for extracting column data from columngroup chunk
+ *
+ * @return
+ * @throws KeyGenException
+ */
+ private KeyStructureInfo getKeyStructureInfo() throws KeyGenException {
+ int colGrpId = getColumnGroupId(dimColEvaluatorInfo.getColumnIndex());
+ KeyGenerator keyGenerator = segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
+ List<Integer> mdKeyOrdinal = new ArrayList<Integer>();
+ mdKeyOrdinal.add(getMdkeyOrdinal(dimColEvaluatorInfo.getColumnIndex(), colGrpId));
+ int[] maskByteRanges = QueryUtil.getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
+ byte[] maxKey = QueryUtil.getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator);
+ KeyStructureInfo restructureInfos = new KeyStructureInfo();
+ restructureInfos.setKeyGenerator(keyGenerator);
+ restructureInfos.setMaskByteRanges(maskByteRanges);
+ restructureInfos.setMaxKey(maxKey);
+ return restructureInfos;
+ }
+
+ /**
+ * Check if scan is required on given block based on min and max value
+ */
+ public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) {
+ BitSet bitSet = new BitSet(1);
+ bitSet.flip(0, 1);
+ return bitSet;
+ }
+
+ private int getMdkeyOrdinal(int ordinal, int colGrpId) {
+ return segmentProperties.getColumnGroupMdKeyOrdinal(colGrpId, ordinal);
+ }
+
+ private int getColumnGroupId(int ordinal) {
+ int[][] columnGroups = segmentProperties.getColumnGroups();
+ int colGrpId = -1;
+ for (int i = 0; i < columnGroups.length; i++) {
+ if (columnGroups[i].length > 1) {
+ colGrpId++;
+ if (QueryUtil.searchInArray(columnGroups[i], ordinal)) {
+ break;
+ }
+ }
+ }
+ return colGrpId;
+ }
+
+ public KeyGenerator getKeyGenerator(int colGrpId) {
+ return segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
new file mode 100644
index 0000000..e8c9903
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter.executer;
+
+import java.io.IOException;
+import java.util.BitSet;
+
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+
+public class ExcludeFilterExecuterImpl implements FilterExecuter {
+
+ protected DimColumnResolvedFilterInfo dimColEvaluatorInfo;
+ protected DimColumnExecuterFilterInfo dimColumnExecuterInfo;
+ protected SegmentProperties segmentProperties;
+
+ public ExcludeFilterExecuterImpl(DimColumnResolvedFilterInfo dimColEvaluatorInfo,
+ SegmentProperties segmentProperties) {
+ this.dimColEvaluatorInfo = dimColEvaluatorInfo;
+ dimColumnExecuterInfo = new DimColumnExecuterFilterInfo();
+ this.segmentProperties = segmentProperties;
+ FilterUtil.prepareKeysFromSurrogates(dimColEvaluatorInfo.getFilterValues(), segmentProperties,
+ dimColEvaluatorInfo.getDimension(), dimColumnExecuterInfo);
+ }
+
+ @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException {
+ int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ .get(dimColEvaluatorInfo.getColumnIndex());
+ if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
+ blockChunkHolder.getDimensionDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
+ return getFilteredIndexes(blockChunkHolder.getDimensionDataChunk()[blockIndex],
+ blockChunkHolder.getDataBlock().nodeSize());
+ }
+
+ protected BitSet getFilteredIndexes(DimensionColumnDataChunk dimColumnDataChunk,
+ int numerOfRows) {
+ // For high cardinality dimensions.
+ if (dimColumnDataChunk.isNoDicitionaryColumn()
+ && dimColumnDataChunk instanceof VariableLengthDimensionDataChunk) {
+ return setDirectKeyFilterIndexToBitSet((VariableLengthDimensionDataChunk) dimColumnDataChunk,
+ numerOfRows);
+ }
+ if (dimColumnDataChunk.isExplicitSorted()
+ && dimColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
+ return setFilterdIndexToBitSetWithColumnIndex(
+ (FixedLengthDimensionDataChunk) dimColumnDataChunk, numerOfRows);
+ }
+ return setFilterdIndexToBitSet((FixedLengthDimensionDataChunk) dimColumnDataChunk, numerOfRows);
+ }
+
+ private BitSet setDirectKeyFilterIndexToBitSet(
+ VariableLengthDimensionDataChunk dimColumnDataChunk, int numerOfRows) {
+ BitSet bitSet = new BitSet(numerOfRows);
+ bitSet.flip(0, numerOfRows);
+ byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
+ for (int i = 0; i < filterValues.length; i++) {
+ byte[] filterVal = filterValues[i];
+ if (dimColumnDataChunk.isExplicitSorted()) {
+ for (int index = 0; index < numerOfRows; index++) {
+ if (dimColumnDataChunk.compareTo(index, filterVal) == 0) {
+ bitSet.flip(dimColumnDataChunk.getInvertedIndex(index));
+ }
+ }
+ } else {
+ for (int index = 0; index < numerOfRows; index++) {
+ if (dimColumnDataChunk.compareTo(index, filterVal) == 0) {
+ bitSet.flip(index);
+ }
+ }
+ }
+ }
+ return bitSet;
+
+ }
+
+ private BitSet setFilterdIndexToBitSetWithColumnIndex(
+ FixedLengthDimensionDataChunk dimColumnDataChunk, int numerOfRows) {
+ int startKey = 0;
+ int last = 0;
+ int startIndex = 0;
+ BitSet bitSet = new BitSet(numerOfRows);
+ bitSet.flip(0, numerOfRows);
+ byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
+ for (int i = 0; i < filterValues.length; i++) {
+ startKey = CarbonUtil
+ .getFirstIndexUsingBinarySearch(dimColumnDataChunk, startIndex, numerOfRows - 1,
+ filterValues[i], false);
+ if (startKey < 0) {
+ continue;
+ }
+ bitSet.flip(dimColumnDataChunk.getInvertedIndex(startKey));
+ last = startKey;
+ for (int j = startKey + 1; j < numerOfRows; j++) {
+ if (dimColumnDataChunk.compareTo(j, filterValues[i]) == 0) {
+ bitSet.flip(dimColumnDataChunk.getInvertedIndex(j));
+ last++;
+ } else {
+ break;
+ }
+ }
+ startIndex = last;
+ if (startIndex >= numerOfRows) {
+ break;
+ }
+ }
+ return bitSet;
+ }
+
+ private BitSet setFilterdIndexToBitSet(FixedLengthDimensionDataChunk dimColumnDataChunk,
+ int numerOfRows) {
+ BitSet bitSet = new BitSet(numerOfRows);
+ bitSet.flip(0, numerOfRows);
+ byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
+ for (int k = 0; k < filterValues.length; k++) {
+ for (int j = 0; j < numerOfRows; j++) {
+ if (dimColumnDataChunk.compareTo(j, filterValues[k]) == 0) {
+ bitSet.flip(j);
+ }
+ }
+ }
+ return bitSet;
+ }
+
+ @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ BitSet bitSet = new BitSet(1);
+ bitSet.flip(0, 1);
+ return bitSet;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
new file mode 100644
index 0000000..4f0a914
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/FilterExecuter.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter.executer;
+
+import java.io.IOException;
+import java.util.BitSet;
+
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+
+public interface FilterExecuter {
+
+ /**
+ * API will apply filter based on resolver instance
+ *
+ * @return
+ * @throws FilterUnsupportedException
+ */
+ BitSet applyFilter(BlocksChunkHolder blocksChunkHolder)
+ throws FilterUnsupportedException, IOException;
+
+ /**
+ * API will verify whether the block can be shortlisted based on block
+ * max and min key.
+ *
+ * @param blockMaxValue, maximum value of the
+ * @param blockMinValue
+ * @return BitSet
+ */
+ BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue);
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
new file mode 100644
index 0000000..22aa3c9
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter.executer;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.keygenerator.KeyGenerator;
+import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.executor.util.QueryUtil;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+
+/**
+ * It checks if filter is required on given block and if required, it does
+ * linear search on block data and set the bitset.
+ */
+public class IncludeColGroupFilterExecuterImpl extends IncludeFilterExecuterImpl {
+
+ /**
+ * LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(IncludeColGroupFilterExecuterImpl.class.getName());
+
+ /**
+ * @param dimColResolvedFilterInfo
+ * @param segmentProperties
+ */
+ public IncludeColGroupFilterExecuterImpl(DimColumnResolvedFilterInfo dimColResolvedFilterInfo,
+ SegmentProperties segmentProperties) {
+ super(dimColResolvedFilterInfo, segmentProperties);
+ }
+
+ /**
+ * It fills BitSet with row index which matches filter key
+ */
+ protected BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ int numerOfRows) {
+ BitSet bitSet = new BitSet(numerOfRows);
+
+ try {
+ KeyStructureInfo keyStructureInfo = getKeyStructureInfo();
+ byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
+ for (int i = 0; i < filterValues.length; i++) {
+ byte[] filterVal = filterValues[i];
+ for (int rowId = 0; rowId < numerOfRows; rowId++) {
+ byte[] colData = new byte[keyStructureInfo.getMaskByteRanges().length];
+ dimensionColumnDataChunk.fillChunkData(colData, 0, rowId, keyStructureInfo);
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterVal, colData) == 0) {
+ bitSet.set(rowId);
+ }
+ }
+ }
+
+ } catch (Exception e) {
+ LOGGER.error(e);
+ }
+
+ return bitSet;
+ }
+
+ /**
+ * It is required for extracting column data from columngroup chunk
+ *
+ * @return
+ * @throws KeyGenException
+ */
+ private KeyStructureInfo getKeyStructureInfo() throws KeyGenException {
+ int colGrpId = getColumnGroupId(dimColumnEvaluatorInfo.getColumnIndex());
+ KeyGenerator keyGenerator = segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
+ List<Integer> mdKeyOrdinal = new ArrayList<Integer>();
+ mdKeyOrdinal.add(getMdkeyOrdinal(dimColumnEvaluatorInfo.getColumnIndex(), colGrpId));
+ int[] maskByteRanges = QueryUtil.getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
+ byte[] maxKey = QueryUtil.getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator);
+ KeyStructureInfo restructureInfos = new KeyStructureInfo();
+ restructureInfos.setKeyGenerator(keyGenerator);
+ restructureInfos.setMaskByteRanges(maskByteRanges);
+ restructureInfos.setMaxKey(maxKey);
+ return restructureInfos;
+ }
+
+ /**
+ * Check if scan is required on given block based on min and max value
+ */
+ public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) {
+ BitSet bitSet = new BitSet(1);
+ byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
+ int columnIndex = dimColumnEvaluatorInfo.getColumnIndex();
+ int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping().get(columnIndex);
+ int[] cols = getAllColumns(columnIndex);
+ byte[] maxValue = getMinMaxData(cols, blkMaxVal[blockIndex], columnIndex);
+ byte[] minValue = getMinMaxData(cols, blkMinVal[blockIndex], columnIndex);
+ boolean isScanRequired = false;
+ for (int k = 0; k < filterValues.length; k++) {
+ // filter value should be in range of max and min value i.e
+ // max>filtervalue>min
+ // so filter-max should be negative
+ int maxCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], maxValue);
+ // and filter-min should be positive
+ int minCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], minValue);
+
+ // if any filter value is in range than this block needs to be
+ // scanned
+ if (maxCompare <= 0 && minCompare >= 0) {
+ isScanRequired = true;
+ break;
+ }
+ }
+ if (isScanRequired) {
+ bitSet.set(0);
+ }
+ return bitSet;
+ }
+
+ /**
+ * It extract min and max data for given column from stored min max value
+ *
+ * @param colGrpColumns
+ * @param minMaxData
+ * @param columnIndex
+ * @return
+ */
+ private byte[] getMinMaxData(int[] colGrpColumns, byte[] minMaxData, int columnIndex) {
+ int startIndex = 0;
+ int endIndex = 0;
+ if (null != colGrpColumns) {
+ for (int i = 0; i < colGrpColumns.length; i++) {
+ int colGrpId = getColumnGroupId(colGrpColumns[i]);
+ int mdKeyOrdinal = getMdkeyOrdinal(colGrpColumns[i], colGrpId);
+ int[] byteRange = getKeyGenerator(colGrpId).getKeyByteOffsets(mdKeyOrdinal);
+ int colSize = 0;
+ for (int j = byteRange[0]; j <= byteRange[1]; j++) {
+ colSize++;
+ }
+ if (colGrpColumns[i] == columnIndex) {
+ endIndex = startIndex + colSize;
+ break;
+ }
+ startIndex += colSize;
+ }
+ }
+ byte[] data = new byte[endIndex - startIndex];
+ System.arraycopy(minMaxData, startIndex, data, 0, data.length);
+ return data;
+ }
+
+ /**
+ * It returns column groups which have provided column ordinal
+ *
+ * @param columnIndex
+ * @return column group array
+ */
+ private int[] getAllColumns(int columnIndex) {
+ int[][] colGroups = segmentProperties.getColumnGroups();
+ for (int i = 0; i < colGroups.length; i++) {
+ if (QueryUtil.searchInArray(colGroups[i], columnIndex)) {
+ return colGroups[i];
+ }
+ }
+ return null;
+ }
+
+ private int getMdkeyOrdinal(int ordinal, int colGrpId) {
+ return segmentProperties.getColumnGroupMdKeyOrdinal(colGrpId, ordinal);
+ }
+
+ private int getColumnGroupId(int ordinal) {
+ int[][] columnGroups = segmentProperties.getColumnGroups();
+ int colGrpId = -1;
+ for (int i = 0; i < columnGroups.length; i++) {
+ if (columnGroups[i].length > 1) {
+ colGrpId++;
+ if (QueryUtil.searchInArray(columnGroups[i], ordinal)) {
+ break;
+ }
+ }
+ }
+ return colGrpId;
+ }
+
+ public KeyGenerator getKeyGenerator(int colGrpId) {
+ return segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
new file mode 100644
index 0000000..4268966
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter.executer;
+
+import java.io.IOException;
+import java.util.BitSet;
+
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
+import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+
+public class IncludeFilterExecuterImpl implements FilterExecuter {
+
+ protected DimColumnResolvedFilterInfo dimColumnEvaluatorInfo;
+ protected DimColumnExecuterFilterInfo dimColumnExecuterInfo;
+ protected SegmentProperties segmentProperties;
+
+ public IncludeFilterExecuterImpl(DimColumnResolvedFilterInfo dimColumnEvaluatorInfo,
+ SegmentProperties segmentProperties) {
+ this.dimColumnEvaluatorInfo = dimColumnEvaluatorInfo;
+ this.segmentProperties = segmentProperties;
+ dimColumnExecuterInfo = new DimColumnExecuterFilterInfo();
+ FilterUtil.prepareKeysFromSurrogates(dimColumnEvaluatorInfo.getFilterValues(),
+ segmentProperties, dimColumnEvaluatorInfo.getDimension(), dimColumnExecuterInfo);
+
+ }
+
+ @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException {
+ int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ .get(dimColumnEvaluatorInfo.getColumnIndex());
+ if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
+ blockChunkHolder.getDimensionDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
+ return getFilteredIndexes(blockChunkHolder.getDimensionDataChunk()[blockIndex],
+ blockChunkHolder.getDataBlock().nodeSize());
+ }
+
+ protected BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ int numerOfRows) {
+ if (dimensionColumnDataChunk.isNoDicitionaryColumn()
+ && dimensionColumnDataChunk instanceof VariableLengthDimensionDataChunk) {
+ return setDirectKeyFilterIndexToBitSet(
+ (VariableLengthDimensionDataChunk) dimensionColumnDataChunk, numerOfRows);
+ } else if (dimensionColumnDataChunk.isExplicitSorted()
+ && dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
+ return setFilterdIndexToBitSetWithColumnIndex(
+ (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, numerOfRows);
+ }
+
+ return setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows);
+ }
+
+ private BitSet setDirectKeyFilterIndexToBitSet(
+ VariableLengthDimensionDataChunk dimensionColumnDataChunk, int numerOfRows) {
+ BitSet bitSet = new BitSet(numerOfRows);
+ byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
+ for (int i = 0; i < filterValues.length; i++) {
+ byte[] filterVal = filterValues[i];
+ if (dimensionColumnDataChunk.isExplicitSorted()) {
+ for (int index = 0; index < numerOfRows; index++) {
+ if (dimensionColumnDataChunk.compareTo(index, filterVal) == 0) {
+ bitSet.set(dimensionColumnDataChunk.getInvertedIndex(index));
+ }
+ }
+ } else {
+ for (int index = 0; index < numerOfRows; index++) {
+ if (dimensionColumnDataChunk.compareTo(index, filterVal) == 0) {
+ bitSet.set(index);
+ }
+ }
+ }
+ }
+ return bitSet;
+
+ }
+
+ private BitSet setFilterdIndexToBitSetWithColumnIndex(
+ FixedLengthDimensionDataChunk dimensionColumnDataChunk, int numerOfRows) {
+ BitSet bitSet = new BitSet(numerOfRows);
+ int start = 0;
+ int last = 0;
+ int startIndex = 0;
+ byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
+ for (int i = 0; i < filterValues.length; i++) {
+ start = CarbonUtil
+ .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ filterValues[i], false);
+ if (start < 0) {
+ continue;
+ }
+ bitSet.set(dimensionColumnDataChunk.getInvertedIndex(start));
+ last = start;
+ for (int j = start + 1; j < numerOfRows; j++) {
+ if (dimensionColumnDataChunk.compareTo(j, filterValues[i]) == 0) {
+ bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j));
+ last++;
+ } else {
+ break;
+ }
+ }
+ startIndex = last;
+ if (startIndex >= numerOfRows) {
+ break;
+ }
+ }
+ return bitSet;
+ }
+
+ private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
+ int numerOfRows) {
+ BitSet bitSet = new BitSet(numerOfRows);
+ if (dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
+ byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
+ for (int k = 0; k < filterValues.length; k++) {
+ for (int j = 0; j < numerOfRows; j++) {
+ if (dimensionColumnDataChunk.compareTo(j, filterValues[k]) == 0) {
+ bitSet.set(j);
+ }
+ }
+ }
+ }
+ return bitSet;
+ }
+
+ public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) {
+ BitSet bitSet = new BitSet(1);
+ byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
+ int columnIndex = dimColumnEvaluatorInfo.getColumnIndex();
+ int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping().get(columnIndex);
+
+ boolean isScanRequired = false;
+ for (int k = 0; k < filterValues.length; k++) {
+ // filter value should be in range of max and min value i.e
+ // max>filtervalue>min
+ // so filter-max should be negative
+ int maxCompare =
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blkMaxVal[blockIndex]);
+ // and filter-min should be positive
+ int minCompare =
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blkMinVal[blockIndex]);
+
+ // if any filter value is in range than this block needs to be
+ // scanned
+ if (maxCompare <= 0 && minCompare >= 0) {
+ isScanRequired = true;
+ break;
+ }
+ }
+ if (isScanRequired) {
+ bitSet.set(0);
+ }
+ return bitSet;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
new file mode 100644
index 0000000..73b6c6a
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/OrFilterExecuterImpl.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter.executer;
+
+import java.io.IOException;
+import java.util.BitSet;
+
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+
+public class OrFilterExecuterImpl implements FilterExecuter {
+
+ private FilterExecuter leftExecuter;
+ private FilterExecuter rightExecuter;
+
+ public OrFilterExecuterImpl(FilterExecuter leftExecuter, FilterExecuter rightExecuter) {
+ this.leftExecuter = leftExecuter;
+ this.rightExecuter = rightExecuter;
+ }
+
+ @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
+ throws FilterUnsupportedException, IOException {
+ BitSet leftFilters = leftExecuter.applyFilter(blockChunkHolder);
+ BitSet rightFilters = rightExecuter.applyFilter(blockChunkHolder);
+ leftFilters.or(rightFilters);
+
+ return leftFilters;
+ }
+
+ @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ BitSet leftFilters = leftExecuter.isScanRequired(blockMaxValue, blockMinValue);
+ BitSet rightFilters = rightExecuter.isScanRequired(blockMaxValue, blockMinValue);
+ leftFilters.or(rightFilters);
+ return leftFilters;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureFilterExecuterImpl.java
new file mode 100644
index 0000000..cc16818
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureFilterExecuterImpl.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter.executer;
+
+import java.util.BitSet;
+
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+
+public class RestructureFilterExecuterImpl implements FilterExecuter {
+
+ DimColumnExecuterFilterInfo dimColumnExecuterInfo;
+
+ public RestructureFilterExecuterImpl(DimColumnResolvedFilterInfo dimColumnResolvedFilterInfo,
+ SegmentProperties segmentProperties) {
+ dimColumnExecuterInfo = new DimColumnExecuterFilterInfo();
+ FilterUtil
+ .prepareKeysFromSurrogates(dimColumnResolvedFilterInfo.getFilterValues(), segmentProperties,
+ dimColumnResolvedFilterInfo.getDimension(), dimColumnExecuterInfo);
+ }
+
+ @Override public BitSet applyFilter(BlocksChunkHolder blocksChunkHolder) {
+ BitSet bitSet = new BitSet(blocksChunkHolder.getDataBlock().nodeSize());
+ byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
+ if (null != filterValues && filterValues.length > 0) {
+ bitSet.set(0, blocksChunkHolder.getDataBlock().nodeSize());
+ }
+ return bitSet;
+ }
+
+ @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ BitSet bitSet = new BitSet(1);
+ bitSet.set(0);
+ return bitSet;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
new file mode 100644
index 0000000..0c1b1a1
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -0,0 +1,368 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter.executer;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.cache.dictionary.Dictionary;
+import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.metadata.Encoding;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.executor.util.QueryUtil;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.GenericQueryType;
+import org.apache.carbondata.core.scan.filter.intf.RowImpl;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+
+public class RowLevelFilterExecuterImpl implements FilterExecuter {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(RowLevelFilterExecuterImpl.class.getName());
+ protected List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList;
+ protected List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList;
+ protected Expression exp;
+ protected AbsoluteTableIdentifier tableIdentifier;
+ protected SegmentProperties segmentProperties;
+ /**
+ * it has index at which given dimension is stored in file
+ */
+ private int[] blocksIndex;
+
+ private Map<Integer, GenericQueryType> complexDimensionInfoMap;
+
+ public RowLevelFilterExecuterImpl(List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
+ List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
+ AbsoluteTableIdentifier tableIdentifier, SegmentProperties segmentProperties,
+ Map<Integer, GenericQueryType> complexDimensionInfoMap) {
+ this.dimColEvaluatorInfoList = dimColEvaluatorInfoList;
+ this.segmentProperties = segmentProperties;
+ this.blocksIndex = new int[dimColEvaluatorInfoList.size()];
+ for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
+ this.blocksIndex[i] = segmentProperties.getDimensionOrdinalToBlockMapping()
+ .get(dimColEvaluatorInfoList.get(i).getColumnIndex());
+ }
+ if (null == msrColEvalutorInfoList) {
+ this.msrColEvalutorInfoList = new ArrayList<MeasureColumnResolvedFilterInfo>(20);
+ } else {
+ this.msrColEvalutorInfoList = msrColEvalutorInfoList;
+ }
+ this.exp = exp;
+ this.tableIdentifier = tableIdentifier;
+ this.complexDimensionInfoMap = complexDimensionInfoMap;
+ }
+
+ @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
+ throws FilterUnsupportedException, IOException {
+ for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
+ DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i);
+ if (dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.ARRAY
+ && dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.STRUCT) {
+ if (null == blockChunkHolder.getDimensionDataChunk()[blocksIndex[i]]) {
+ blockChunkHolder.getDimensionDataChunk()[blocksIndex[i]] = blockChunkHolder.getDataBlock()
+ .getDimensionChunk(blockChunkHolder.getFileReader(), blocksIndex[i]);
+ }
+ } else {
+ GenericQueryType complexType = complexDimensionInfoMap.get(blocksIndex[i]);
+ complexType.fillRequiredBlockData(blockChunkHolder);
+ }
+ }
+
+ // CHECKSTYLE:OFF Approval No:Approval-V1R2C10_001
+ if (null != msrColEvalutorInfoList) {
+ for (MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo : msrColEvalutorInfoList) {
+ if (null == blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo
+ .getColumnIndex()]) {
+ blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo.getColumnIndex()] =
+ blockChunkHolder.getDataBlock().getMeasureChunk(blockChunkHolder.getFileReader(),
+ msrColumnEvalutorInfo.getColumnIndex());
+ }
+ }
+ }
+ // CHECKSTYLE:ON
+
+ int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
+ BitSet set = new BitSet(numberOfRows);
+ RowIntf row = new RowImpl();
+ boolean invalidRowsPresent = false;
+ for (int index = 0; index < numberOfRows; index++) {
+ createRow(blockChunkHolder, row, index);
+ Boolean rslt = false;
+ try {
+ rslt = exp.evaluate(row).getBoolean();
+ }
+ // Any invalid member while evaluation shall be ignored, system will log the
+ // error only once since all rows the evaluation happens so inorder to avoid
+ // too much log inforation only once the log will be printed.
+ catch (FilterIllegalMemberException e) {
+ FilterUtil.logError(e, invalidRowsPresent);
+ }
+ if (null != rslt && rslt) {
+ set.set(index);
+ }
+ }
+ return set;
+ }
+
+ /**
+ * Method will read the members of particular dimension block and create
+ * a row instance for further processing of the filters
+ *
+ * @param blockChunkHolder
+ * @param row
+ * @param index
+ * @throws IOException
+ */
+ private void createRow(BlocksChunkHolder blockChunkHolder, RowIntf row, int index)
+ throws IOException {
+ Object[] record = new Object[dimColEvaluatorInfoList.size() + msrColEvalutorInfoList.size()];
+ String memberString;
+ for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
+ DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i);
+ if (dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.ARRAY
+ && dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.STRUCT) {
+ if (!dimColumnEvaluatorInfo.isDimensionExistsInCurrentSilce()) {
+ record[dimColumnEvaluatorInfo.getRowIndex()] = dimColumnEvaluatorInfo.getDefaultValue();
+ }
+ if (!dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)
+ && blockChunkHolder
+ .getDimensionDataChunk()[blocksIndex[i]] instanceof VariableLengthDimensionDataChunk) {
+
+ VariableLengthDimensionDataChunk dimensionColumnDataChunk =
+ (VariableLengthDimensionDataChunk) blockChunkHolder
+ .getDimensionDataChunk()[blocksIndex[i]];
+ memberString = readMemberBasedOnNoDictionaryVal(dimensionColumnDataChunk, index);
+ if (null != memberString) {
+ if (memberString.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
+ memberString = null;
+ }
+ record[dimColumnEvaluatorInfo.getRowIndex()] = DataTypeUtil
+ .getDataBasedOnDataType(memberString,
+ dimColumnEvaluatorInfo.getDimension().getDataType());
+ } else {
+ continue;
+ }
+ } else {
+ int dictionaryValue =
+ readSurrogatesFromColumnBlock(blockChunkHolder, index, dimColumnEvaluatorInfo,
+ blocksIndex[i]);
+ if (dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)
+ && !dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ memberString =
+ getFilterActualValueFromDictionaryValue(dimColumnEvaluatorInfo, dictionaryValue);
+ record[dimColumnEvaluatorInfo.getRowIndex()] = DataTypeUtil
+ .getDataBasedOnDataType(memberString,
+ dimColumnEvaluatorInfo.getDimension().getDataType());
+ } else if (dimColumnEvaluatorInfo.getDimension()
+ .hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+
+ Object member = getFilterActualValueFromDirectDictionaryValue(dimColumnEvaluatorInfo,
+ dictionaryValue);
+ record[dimColumnEvaluatorInfo.getRowIndex()] = member;
+ }
+ }
+ } else {
+ try {
+ GenericQueryType complexType = complexDimensionInfoMap.get(blocksIndex[i]);
+ ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+ DataOutputStream dataOutputStream = new DataOutputStream(byteStream);
+ complexType
+ .parseBlocksAndReturnComplexColumnByteArray(blockChunkHolder.getDimensionDataChunk(),
+ index, dataOutputStream);
+ record[dimColumnEvaluatorInfo.getRowIndex()] = complexType
+ .getDataBasedOnDataTypeFromSurrogates(ByteBuffer.wrap(byteStream.toByteArray()));
+ byteStream.close();
+ } catch (IOException e) {
+ LOGGER.info(e.getMessage());
+ }
+ }
+ }
+
+ DataType msrType;
+
+ for (MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo : msrColEvalutorInfoList) {
+ switch (msrColumnEvalutorInfo.getType()) {
+ case INT:
+ case LONG:
+ msrType = DataType.LONG;
+ break;
+ case DECIMAL:
+ msrType = DataType.DECIMAL;
+ break;
+ default:
+ msrType = DataType.DOUBLE;
+ }
+ // if measure doesnt exist then set the default value.
+ Object msrValue;
+ switch (msrType) {
+ case INT:
+ case LONG:
+ msrValue = blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo.getColumnIndex()]
+ .getMeasureDataHolder().getReadableLongValueByIndex(index);
+ break;
+ case DECIMAL:
+ msrValue = blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo.getColumnIndex()]
+ .getMeasureDataHolder().getReadableBigDecimalValueByIndex(index);
+ break;
+ default:
+ msrValue = blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo.getColumnIndex()]
+ .getMeasureDataHolder().getReadableDoubleValueByIndex(index);
+ }
+ record[msrColumnEvalutorInfo.getRowIndex()] =
+ blockChunkHolder.getMeasureDataChunk()[msrColumnEvalutorInfo.getColumnIndex()]
+ .getNullValueIndexHolder().getBitSet().get(index) ? null : msrValue;
+ }
+ row.setValues(record);
+ }
+
+ /**
+ * method will read the actual data from the direct dictionary generator
+ * by passing direct dictionary value.
+ *
+ * @param dimColumnEvaluatorInfo
+ * @param dictionaryValue
+ * @return
+ */
+ private Object getFilterActualValueFromDirectDictionaryValue(
+ DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, int dictionaryValue) {
+ Object memberString = null;
+ DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
+ .getDirectDictionaryGenerator(dimColumnEvaluatorInfo.getDimension().getDataType());
+ if (null != directDictionaryGenerator) {
+ memberString = directDictionaryGenerator.getValueFromSurrogate(dictionaryValue);
+ }
+ return memberString;
+ }
+
+ /**
+ * Read the actual filter member by passing the dictionary value from
+ * the forward dictionary cache which which holds column wise cache
+ *
+ * @param dimColumnEvaluatorInfo
+ * @param dictionaryValue
+ * @return
+ * @throws IOException
+ */
+ private String getFilterActualValueFromDictionaryValue(
+ DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, int dictionaryValue) throws IOException {
+ String memberString;
+ Dictionary forwardDictionary = FilterUtil
+ .getForwardDictionaryCache(tableIdentifier, dimColumnEvaluatorInfo.getDimension());
+
+ memberString = forwardDictionary.getDictionaryValueForKey(dictionaryValue);
+ if (null != memberString) {
+ if (memberString.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
+ memberString = null;
+ }
+ }
+ return memberString;
+ }
+
+ /**
+ * read the filter member dictionary data from the block corresponding to
+ * applied filter column
+ *
+ * @param blockChunkHolder
+ * @param index
+ * @param dimColumnEvaluatorInfo
+ * @return
+ */
+ private int readSurrogatesFromColumnBlock(BlocksChunkHolder blockChunkHolder, int index,
+ DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, int blockIndex) {
+ if (dimColumnEvaluatorInfo.getDimension().isColumnar()) {
+ byte[] rawData = blockChunkHolder.getDimensionDataChunk()[blockIndex].getChunkData(index);
+ ByteBuffer byteBuffer = ByteBuffer.allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE);
+ int dictionaryValue = CarbonUtil.getSurrogateKey(rawData, byteBuffer);
+ return dictionaryValue;
+ } else {
+ return readSurrogatesFromColumnGroupBlock(blockChunkHolder, index, dimColumnEvaluatorInfo,
+ blockIndex);
+ }
+
+ }
+
+ /**
+ * @param blockChunkHolder
+ * @param index
+ * @param dimColumnEvaluatorInfo
+ * @return read surrogate of given row of given column group dimension
+ */
+ private int readSurrogatesFromColumnGroupBlock(BlocksChunkHolder blockChunkHolder, int index,
+ DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, int blockIndex) {
+ try {
+ KeyStructureInfo keyStructureInfo =
+ QueryUtil.getKeyStructureInfo(segmentProperties, dimColumnEvaluatorInfo);
+ byte[] colData = blockChunkHolder.getDimensionDataChunk()[blockIndex].getChunkData(index);
+ long[] result = keyStructureInfo.getKeyGenerator().getKeyArray(colData);
+ int colGroupId =
+ QueryUtil.getColumnGroupId(segmentProperties, dimColumnEvaluatorInfo.getColumnIndex());
+ int dictionaryValue = (int) result[segmentProperties
+ .getColumnGroupMdKeyOrdinal(colGroupId, dimColumnEvaluatorInfo.getColumnIndex())];
+ return dictionaryValue;
+ } catch (KeyGenException e) {
+ LOGGER.error(e);
+ }
+ return 0;
+ }
+
+ /**
+ * Reading the blocks for no dictionary data, in no dictionary case
+ * directly the filter data will read, no need to scan the dictionary
+ * or read the dictionary value.
+ *
+ * @param dimensionColumnDataChunk
+ * @param index
+ * @return
+ */
+ private String readMemberBasedOnNoDictionaryVal(
+ VariableLengthDimensionDataChunk dimensionColumnDataChunk, int index) {
+ return new String(dimensionColumnDataChunk.getChunkData(index),
+ Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+ }
+
+ @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ BitSet bitSet = new BitSet(1);
+ bitSet.set(0);
+ return bitSet;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
new file mode 100644
index 0000000..11ec991
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter.executer;
+
+import java.io.IOException;
+import java.util.BitSet;
+import java.util.List;
+
+import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
+import org.apache.carbondata.core.metadata.Encoding;
+import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+
+public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecuterImpl {
+ private byte[][] filterRangeValues;
+
+ public RowLevelRangeGrtThanFiterExecuterImpl(
+ List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
+ List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
+ AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues,
+ SegmentProperties segmentProperties) {
+ super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties,
+ null);
+ this.filterRangeValues = filterRangeValues;
+ }
+
+ @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ BitSet bitSet = new BitSet(1);
+ byte[][] filterValues = this.filterRangeValues;
+ int columnIndex = this.dimColEvaluatorInfoList.get(0).getColumnIndex();
+ boolean isScanRequired = false;
+ for (int k = 0; k < filterValues.length; k++) {
+ // filter value should be in range of max and min value i.e
+ // max>filtervalue>min
+ // so filter-max should be negative
+ int maxCompare =
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue[columnIndex]);
+ // if any filter value is in range than this block needs to be
+ // scanned means always less than block max range.
+ if (maxCompare < 0) {
+ isScanRequired = true;
+ break;
+ }
+ }
+ if (isScanRequired) {
+ bitSet.set(0);
+ }
+ return bitSet;
+
+ }
+
+ @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
+ throws FilterUnsupportedException, IOException {
+ if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ return super.applyFilter(blockChunkHolder);
+ }
+ int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ .get(dimColEvaluatorInfoList.get(0).getColumnIndex());
+ if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
+ blockChunkHolder.getDimensionDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
+ return getFilteredIndexes(blockChunkHolder.getDimensionDataChunk()[blockIndex],
+ blockChunkHolder.getDataBlock().nodeSize());
+ }
+
+ private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ int numerOfRows) {
+ if (dimensionColumnDataChunk.isExplicitSorted()
+ && dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
+ return setFilterdIndexToBitSetWithColumnIndex(
+ (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, numerOfRows);
+ }
+ return setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows);
+ }
+
+ /**
+ * Method will scan the block and finds the range start index from which all members
+ * will be considered for applying range filters. this method will be called if the
+ * column is not supported by default so column index mapping will be present for
+ * accesing the members from the block.
+ *
+ * @param dimensionColumnDataChunk
+ * @param numerOfRows
+ * @return BitSet.
+ */
+ private BitSet setFilterdIndexToBitSetWithColumnIndex(
+ FixedLengthDimensionDataChunk dimensionColumnDataChunk, int numerOfRows) {
+ BitSet bitSet = new BitSet(numerOfRows);
+ int start = 0;
+ int last = 0;
+ int startIndex = 0;
+ byte[][] filterValues = this.filterRangeValues;
+ for (int i = 0; i < filterValues.length; i++) {
+ start = CarbonUtil
+ .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ filterValues[i], true);
+ if (start >= 0) {
+ start = CarbonUtil.nextGreaterValueToTarget(start, dimensionColumnDataChunk,
+ filterValues[i], numerOfRows);
+ }
+ // Logic will handle the case where the range filter member is not present in block
+ // in this case the binary search will return the index from where the bit sets will be
+ // set inorder to apply filters. this is greater than filter so the range will be taken
+ // from the next element which is greater than filter member.
+ if (start < 0) {
+ start = -(start + 1);
+ if (start == numerOfRows) {
+ start = start - 1;
+ }
+ // Method will compare the tentative index value after binary search, this tentative
+ // index needs to be compared by the filter member if its > filter then from that
+ // index the bitset will be considered for filtering process.
+ if (ByteUtil.compare(filterValues[i],
+ dimensionColumnDataChunk.getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))
+ > 0) {
+ start = start + 1;
+ }
+ }
+
+ last = start;
+ for (int j = start; j < numerOfRows; j++) {
+ bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j));
+ last++;
+ }
+ startIndex = last;
+ if (startIndex >= numerOfRows) {
+ break;
+ }
+ }
+
+ return bitSet;
+ }
+
+ /**
+ * Method will scan the block and finds the range start index from which all
+ * members will be considered for applying range filters. this method will
+ * be called if the column is sorted default so column index
+ * mapping will be present for accesing the members from the block.
+ *
+ * @param dimensionColumnDataChunk
+ * @param numerOfRows
+ * @return BitSet.
+ */
+ private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
+ int numerOfRows) {
+ BitSet bitSet = new BitSet(numerOfRows);
+ if (dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
+ int start = 0;
+ int last = 0;
+ int startIndex = 0;
+ byte[][] filterValues = this.filterRangeValues;
+ for (int k = 0; k < filterValues.length; k++) {
+ start = CarbonUtil.getFirstIndexUsingBinarySearch(
+ (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ filterValues[k], true);
+ if(start >= 0){
+ start = CarbonUtil.nextGreaterValueToTarget(start,
+ (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, filterValues[k],
+ numerOfRows);
+ }
+ if (start < 0) {
+ start = -(start + 1);
+ if (start == numerOfRows) {
+ start = start - 1;
+ }
+ // Method will compare the tentative index value after binary search, this tentative
+ // index needs to be compared by the filter member if its > filter then from that
+ // index the bitset will be considered for filtering process.
+ if (ByteUtil.compare(filterValues[k], dimensionColumnDataChunk.getChunkData(start)) > 0) {
+ start = start + 1;
+ }
+ }
+ last = start;
+ for (int j = start; j < numerOfRows; j++) {
+ bitSet.set(j);
+ last++;
+ }
+ startIndex = last;
+ if (startIndex >= numerOfRows) {
+ break;
+ }
+ }
+ }
+ return bitSet;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
new file mode 100644
index 0000000..fb7c86d
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter.executer;
+
+import java.io.IOException;
+import java.util.BitSet;
+import java.util.List;
+
+import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
+import org.apache.carbondata.core.metadata.Encoding;
+import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+
+public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilterExecuterImpl {
+
+ protected byte[][] filterRangeValues;
+
+ public RowLevelRangeGrtrThanEquaToFilterExecuterImpl(
+ List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
+ List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
+ AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues,
+ SegmentProperties segmentProperties) {
+ super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties,
+ null);
+ this.filterRangeValues = filterRangeValues;
+ }
+
+ @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ BitSet bitSet = new BitSet(1);
+ byte[][] filterValues = this.filterRangeValues;
+ int columnIndex = this.dimColEvaluatorInfoList.get(0).getColumnIndex();
+ boolean isScanRequired = false;
+ for (int k = 0; k < filterValues.length; k++) {
+ // filter value should be in range of max and min value i.e
+ // max>filtervalue>min
+ // so filter-max should be negative
+ int maxCompare =
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue[columnIndex]);
+ // if any filter value is in range than this block needs to be
+ // scanned less than equal to max range.
+ if (maxCompare <= 0) {
+ isScanRequired = true;
+ break;
+ }
+ }
+ if (isScanRequired) {
+ bitSet.set(0);
+ }
+ return bitSet;
+
+ }
+
+ @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
+ throws FilterUnsupportedException, IOException {
+ if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ return super.applyFilter(blockChunkHolder);
+ }
+ int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ .get(dimColEvaluatorInfoList.get(0).getColumnIndex());
+ if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
+ blockChunkHolder.getDimensionDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
+ return getFilteredIndexes(blockChunkHolder.getDimensionDataChunk()[blockIndex],
+ blockChunkHolder.getDataBlock().nodeSize());
+ }
+
+ private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ int numerOfRows) {
+ if (dimensionColumnDataChunk.isExplicitSorted()
+ && dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
+ return setFilterdIndexToBitSetWithColumnIndex(
+ (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, numerOfRows);
+ }
+ return setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows);
+ }
+
+ /**
+ * Method will scan the block and finds the range start index from which all members
+ * will be considered for applying range filters. this method will be called if the
+ * column is not supported by default so column index mapping will be present for
+ * accesing the members from the block.
+ *
+ * @param dimensionColumnDataChunk
+ * @param numerOfRows
+ * @return BitSet.
+ */
+ private BitSet setFilterdIndexToBitSetWithColumnIndex(
+ FixedLengthDimensionDataChunk dimensionColumnDataChunk, int numerOfRows) {
+ BitSet bitSet = new BitSet(numerOfRows);
+ int start = 0;
+ int last = 0;
+ int startIndex = 0;
+ byte[][] filterValues = this.filterRangeValues;
+ for (int i = 0; i < filterValues.length; i++) {
+ start = CarbonUtil
+ .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ filterValues[i], false);
+ if (start < 0) {
+ start = -(start + 1);
+ if (start == numerOfRows) {
+ start = start - 1;
+ }
+ // Method will compare the tentative index value after binary search, this tentative
+ // index needs to be compared by the filter member if its >= filter then from that
+ // index the bitset will be considered for filtering process.
+ if (ByteUtil.compare(filterValues[i],
+ dimensionColumnDataChunk.getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))
+ >= 0) {
+ start = start + 1;
+ }
+ }
+ last = start;
+ for (int j = start; j < numerOfRows; j++) {
+ bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j));
+ last++;
+ }
+ startIndex = last;
+ if (startIndex >= numerOfRows) {
+ break;
+ }
+ }
+ return bitSet;
+ }
+
+ /**
+ * Method will scan the block and finds the range start index from which all
+ * members will be considered for applying range filters. this method will
+ * be called if the column is sorted default so column index
+ * mapping will be present for accesing the members from the block.
+ *
+ * @param dimensionColumnDataChunk
+ * @param numerOfRows
+ * @return BitSet.
+ */
+ private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
+ int numerOfRows) {
+ BitSet bitSet = new BitSet(numerOfRows);
+ if (dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
+ int start = 0;
+ int last = 0;
+ int startIndex = 0;
+ byte[][] filterValues = this.filterRangeValues;
+ for (int k = 0; k < filterValues.length; k++) {
+ start = CarbonUtil.getFirstIndexUsingBinarySearch(
+ (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ filterValues[k], false);
+ if (start < 0) {
+ start = -(start + 1);
+ if (start == numerOfRows) {
+ start = start - 1;
+ }
+ // Method will compare the tentative index value after binary search, this tentative
+ // index needs to be compared by the filter member if its >= filter then from that
+ // index the bitset will be considered for filtering process.
+ if (ByteUtil.compare(filterValues[k], dimensionColumnDataChunk.getChunkData(start))
+ >= 0) {
+ start = start + 1;
+ }
+ }
+
+ last = start;
+ for (int j = start; j < numerOfRows; j++) {
+ bitSet.set(j);
+ last++;
+ }
+ startIndex = last;
+ if (startIndex >= numerOfRows) {
+ break;
+ }
+ }
+ }
+ return bitSet;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
new file mode 100644
index 0000000..33c2e50
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter.executer;
+
+import java.io.IOException;
+import java.util.BitSet;
+import java.util.List;
+
+import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
+import org.apache.carbondata.core.metadata.Encoding;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
+import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+
+public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilterExecuterImpl {
+ protected byte[][] filterRangeValues;
+
+ public RowLevelRangeLessThanEqualFilterExecuterImpl(
+ List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
+ List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
+ AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues,
+ SegmentProperties segmentProperties) {
+ super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties,
+ null);
+ this.filterRangeValues = filterRangeValues;
+ }
+
+ @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ BitSet bitSet = new BitSet(1);
+ byte[][] filterValues = this.filterRangeValues;
+ int columnIndex = this.dimColEvaluatorInfoList.get(0).getColumnIndex();
+ boolean isScanRequired = false;
+ for (int k = 0; k < filterValues.length; k++) {
+ // and filter-min should be positive
+ int minCompare =
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMinValue[columnIndex]);
+
+ // if any filter applied is not in range of min and max of block
+ // then since its a less than equal to fiter validate whether the block
+ // min range is less than equal to applied filter member
+ if (minCompare >= 0) {
+ isScanRequired = true;
+ break;
+ }
+ }
+ if (isScanRequired) {
+ bitSet.set(0);
+ }
+ return bitSet;
+
+ }
+
+ @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
+ throws FilterUnsupportedException, IOException {
+ if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ return super.applyFilter(blockChunkHolder);
+ }
+ int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ .get(dimColEvaluatorInfoList.get(0).getColumnIndex());
+ if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
+ blockChunkHolder.getDimensionDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
+ return getFilteredIndexes(blockChunkHolder.getDimensionDataChunk()[blockIndex],
+ blockChunkHolder.getDataBlock().nodeSize());
+ }
+
+ private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ int numerOfRows) {
+ byte[] defaultValue = null;
+ if (dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
+ .getDirectDictionaryGenerator(
+ dimColEvaluatorInfoList.get(0).getDimension().getDataType());
+ int key = directDictionaryGenerator.generateDirectSurrogateKey(null) + 1;
+ defaultValue = FilterUtil.getMaskKey(key, dimColEvaluatorInfoList.get(0).getDimension(),
+ this.segmentProperties.getDimensionKeyGenerator());
+ }
+ if (dimensionColumnDataChunk.isExplicitSorted()
+ && dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
+
+ return setFilterdIndexToBitSetWithColumnIndex(
+ (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, numerOfRows, defaultValue);
+
+ }
+ return setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows, defaultValue);
+ }
+
+ /**
+ * Method will scan the block and finds the range start index from which all members
+ * will be considered for applying range filters. this method will be called if the
+ * column is not supported by default so column index mapping will be present for
+ * accesing the members from the block.
+ *
+ * @param dimensionColumnDataChunk
+ * @param numerOfRows
+ * @return BitSet.
+ */
+ private BitSet setFilterdIndexToBitSetWithColumnIndex(
+ FixedLengthDimensionDataChunk dimensionColumnDataChunk, int numerOfRows,
+ byte[] defaultValue) {
+ BitSet bitSet = new BitSet(numerOfRows);
+ int start = 0;
+ int last = 0;
+ int skip = 0;
+ int startIndex = 0;
+ byte[][] filterValues = this.filterRangeValues;
+ //find the number of default values to skip the null value in case of direct dictionary
+ if (null != defaultValue) {
+ start = CarbonUtil
+ .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ defaultValue, true);
+ if (start < 0) {
+ skip = -(start + 1);
+ // end of block
+ if (skip == numerOfRows) {
+ return bitSet;
+ }
+ } else {
+ skip = start;
+ }
+ startIndex = skip;
+ }
+ for (int i = 0; i < filterValues.length; i++) {
+ start = CarbonUtil
+ .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ filterValues[i], true);
+ if (start < 0) {
+ start = -(start + 1);
+ if (start == numerOfRows) {
+ start = start - 1;
+ }
+ // Method will compare the tentative index value after binary search, this tentative
+ // index needs to be compared by the filter member if its >= filter then from that
+ // index the bitset will be considered for filtering process.
+ if (ByteUtil.compare(filterValues[i],
+ dimensionColumnDataChunk.getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))
+ <= 0) {
+ start = start - 1;
+ }
+ }
+ last = start;
+ for (int j = start; j >= skip; j--) {
+ bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j));
+ last--;
+ }
+ startIndex = last;
+ if (startIndex <= 0) {
+ break;
+ }
+ }
+ return bitSet;
+ }
+
+ /**
+ * Method will scan the block and finds the range start index from which all
+ * members will be considered for applying range filters. this method will
+ * be called if the column is sorted default so column index
+ * mapping will be present for accesing the members from the block.
+ *
+ * @param dimensionColumnDataChunk
+ * @param numerOfRows
+ * @param defaultValue
+ * @return BitSet.
+ */
+ private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
+ int numerOfRows, byte[] defaultValue) {
+ BitSet bitSet = new BitSet(numerOfRows);
+ if (dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
+ int start = 0;
+ int last = 0;
+ int startIndex = 0;
+ byte[][] filterValues = this.filterRangeValues;
+ int skip = 0;
+ //find the number of default values to skip the null value in case of direct dictionary
+ if (null != defaultValue) {
+ start = CarbonUtil.getFirstIndexUsingBinarySearch(
+ (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ defaultValue, true);
+ if (start < 0) {
+ skip = -(start + 1);
+ // end of block
+ if (skip == numerOfRows) {
+ return bitSet;
+ }
+ } else {
+ skip = start;
+ }
+ startIndex = skip;
+ }
+ for (int k = 0; k < filterValues.length; k++) {
+ start = CarbonUtil.getFirstIndexUsingBinarySearch(
+ (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ filterValues[k], true);
+ if (start < 0) {
+ start = -(start + 1);
+ if (start == numerOfRows) {
+ start = start - 1;
+ }
+ // Method will compare the tentative index value after binary search, this tentative
+ // index needs to be compared by the filter member if its <= filter then from that
+ // index the bitset will be considered for filtering process.
+ if (ByteUtil.compare(filterValues[k], dimensionColumnDataChunk.getChunkData(start))
+ <= 0) {
+ start = start - 1;
+ }
+ }
+ last = start;
+ for (int j = start; j >= skip; j--) {
+ bitSet.set(j);
+ last--;
+ }
+ startIndex = last;
+ if (startIndex <= 0) {
+ break;
+ }
+ }
+ }
+ return bitSet;
+ }
+
+}
[39/57] [abbrv] incubator-carbondata git commit: move identifier to
metadata package
Posted by ja...@apache.org.
move identifier to metadata package
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/d0d25c9b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/d0d25c9b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/d0d25c9b
Branch: refs/heads/movecorepackage
Commit: d0d25c9bda96b783b943988725733a85be627efe
Parents: 952cf51
Author: jackylk <ja...@huawei.com>
Authored: Fri Jan 13 15:18:44 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Fri Jan 13 15:18:44 2017 +0800
----------------------------------------------------------------------
.../core/AbsoluteTableIdentifier.java | 141 --------------
.../carbondata/core/CarbonDataLoadSchema.java | 184 -----------------
.../carbondata/core/CarbonTableIdentifier.java | 139 -------------
.../carbondata/core/ColumnIdentifier.java | 114 -----------
.../carbondata/core/ColumnarFormatVersion.java | 52 -----
.../core/atomic/AtomicFileOperations.java | 33 ++++
.../core/atomic/AtomicFileOperationsImpl.java | 84 ++++++++
.../core/atomic/FileWriteOperation.java | 25 +++
.../cache/dictionary/DictionaryCacheLoader.java | 2 +-
.../dictionary/DictionaryCacheLoaderImpl.java | 4 +-
.../DictionaryColumnUniqueIdentifier.java | 4 +-
.../core/datastore/BlockIndexStore.java | 4 +-
.../core/datastore/SegmentTaskIndexStore.java | 2 +-
.../datastore/TableSegmentUniqueIdentifier.java | 4 +-
.../core/datastore/block/TableBlockInfo.java | 2 +-
.../block/TableBlockUniqueIdentifier.java | 4 +-
.../chunk/reader/CarbonDataReaderFactory.java | 2 +-
.../IncrementalColumnDictionaryGenerator.java | 4 +-
.../core/metadata/AbsoluteTableIdentifier.java | 140 +++++++++++++
.../core/metadata/CarbonTableIdentifier.java | 139 +++++++++++++
.../core/metadata/ColumnarFormatVersion.java | 52 +++++
.../core/metadata/blocklet/DataFileFooter.java | 2 +-
.../core/metadata/schema/table/CarbonTable.java | 4 +-
.../schema/table/column/CarbonColumn.java | 2 +-
.../carbondata/core/path/CarbonStorePath.java | 4 +-
.../carbondata/core/path/CarbonTablePath.java | 2 +-
.../reader/CarbonDeleteDeltaFileReaderImpl.java | 4 +-
.../CarbonDictionaryMetadataReaderImpl.java | 4 +-
.../core/reader/CarbonDictionaryReaderImpl.java | 4 +-
.../CarbonDictionarySortIndexReaderImpl.java | 4 +-
.../executor/impl/AbstractQueryExecutor.java | 2 +-
.../scan/executor/infos/BlockExecutionInfo.java | 2 +-
.../core/scan/executor/util/QueryUtil.java | 4 +-
.../scan/filter/FilterExpressionProcessor.java | 2 +-
.../core/scan/filter/FilterProcessor.java | 2 +-
.../carbondata/core/scan/filter/FilterUtil.java | 2 +-
.../executer/RowLevelFilterExecuterImpl.java | 2 +-
.../RowLevelRangeGrtThanFiterExecuterImpl.java | 2 +-
...elRangeGrtrThanEquaToFilterExecuterImpl.java | 2 +-
...velRangeLessThanEqualFilterExecuterImpl.java | 2 +-
.../RowLevelRangeLessThanFiterExecuterImpl.java | 2 +-
.../resolver/ConditionalFilterResolverImpl.java | 2 +-
.../filter/resolver/FilterResolverIntf.java | 2 +-
.../resolver/LogicalFilterResolverImpl.java | 2 +-
.../resolver/RowLevelFilterResolverImpl.java | 2 +-
.../RowLevelRangeFilterResolverImpl.java | 2 +-
.../metadata/FilterResolverMetadata.java | 2 +-
.../carbondata/core/scan/model/QueryModel.java | 2 +-
.../core/service/DictionaryService.java | 4 +-
.../carbondata/core/service/PathService.java | 2 +-
.../core/service/impl/DictionaryFactory.java | 4 +-
.../core/service/impl/PathFactory.java | 2 +-
.../core/update/CarbonUpdateUtil.java | 10 +-
.../data/BlockletDeleteDeltaCacheLoader.java | 2 +-
.../core/updatestatus/SegmentStatusManager.java | 18 +-
.../SegmentUpdateStatusManager.java | 14 +-
.../updatestatus/locks/AbstractCarbonLock.java | 77 ++++++++
.../updatestatus/locks/CarbonLockFactory.java | 94 +++++++++
.../core/updatestatus/locks/CarbonLockUtil.java | 64 ++++++
.../core/updatestatus/locks/HdfsFileLock.java | 129 ++++++++++++
.../core/updatestatus/locks/ICarbonLock.java | 40 ++++
.../core/updatestatus/locks/LocalFileLock.java | 164 ++++++++++++++++
.../core/updatestatus/locks/LockUsage.java | 36 ++++
.../updatestatus/locks/ZooKeeperLocking.java | 195 +++++++++++++++++++
.../core/updatestatus/locks/ZookeeperInit.java | 82 ++++++++
.../util/AbstractDataFileFooterConverter.java | 2 +-
.../core/util/CarbonMetadataUtil.java | 2 +-
.../carbondata/core/util/CarbonProperties.java | 2 +-
.../apache/carbondata/core/util/CarbonUtil.java | 2 +-
.../core/util/DataFileFooterConverter.java | 2 +-
.../core/util/DataFileFooterConverter2.java | 2 +-
.../util/DataFileFooterConverterFactory.java | 2 +-
.../core/writer/CarbonDictionaryWriterImpl.java | 4 +-
.../CarbonDictionarySortIndexWriterImpl.java | 4 +-
.../fileoperations/AtomicFileOperations.java | 33 ----
.../AtomicFileOperationsImpl.java | 87 ---------
.../fileoperations/FileWriteOperation.java | 25 ---
.../carbondata/locks/AbstractCarbonLock.java | 77 --------
.../carbondata/locks/CarbonLockFactory.java | 94 ---------
.../apache/carbondata/locks/CarbonLockUtil.java | 64 ------
.../apache/carbondata/locks/HdfsFileLock.java | 129 ------------
.../apache/carbondata/locks/ICarbonLock.java | 40 ----
.../apache/carbondata/locks/LocalFileLock.java | 164 ----------------
.../org/apache/carbondata/locks/LockUsage.java | 36 ----
.../carbondata/locks/ZooKeeperLocking.java | 195 -------------------
.../apache/carbondata/locks/ZookeeperInit.java | 82 --------
.../core/AbsoluteTableIdentifierTest.java | 4 +-
.../core/CarbonTableIdentifierTest.java | 2 +-
.../carbondata/core/ColumnIdentifierTest.java | 2 +-
.../dictionary/AbstractDictionaryCacheTest.java | 4 +-
.../DictionaryCacheLoaderImplTest.java | 4 +-
.../DictionaryColumnUniqueIdentifierTest.java | 4 +-
.../dictionary/ForwardDictionaryCacheTest.java | 4 +-
.../dictionary/ReverseDictionaryCacheTest.java | 4 +-
.../datastore/SegmentTaskIndexStoreTest.java | 6 +-
.../core/datastore/block/BlockInfoTest.java | 2 +-
.../datastore/block/TableBlockInfoTest.java | 2 +-
.../core/datastore/block/TableTaskInfoTest.java | 2 +-
.../CarbonFormatDirectoryStructureTest.java | 5 +-
.../reader/CarbonDictionaryReaderImplTest.java | 4 +-
...CarbonDictionarySortIndexReaderImplTest.java | 4 +-
.../core/scan/filter/FilterUtilTest.java | 4 +-
.../updatestatus/locks/LocalFileLockTest.java | 64 ++++++
.../locks/ZooKeeperLockingTest.java | 142 ++++++++++++++
.../carbondata/core/util/CarbonUtilTest.java | 2 +-
.../core/util/DataFileFooterConverterTest.java | 2 +-
.../writer/CarbonDictionaryWriterImplTest.java | 4 +-
...CarbonDictionarySortIndexWriterImplTest.java | 4 +-
.../examples/GenerateDictionaryExample.scala | 2 +-
.../carbondata/hadoop/CarbonInputFormat.java | 4 +-
.../carbondata/hadoop/CarbonInputSplit.java | 2 +-
.../hadoop/CarbonMultiBlockSplit.java | 2 +-
.../internal/index/impl/InMemoryBTreeIndex.java | 2 +-
.../hadoop/readsupport/CarbonReadSupport.java | 2 +-
.../AbstractDictionaryDecodedReadSupport.java | 2 +-
.../impl/ArrayWritableReadSupport.java | 2 +-
.../readsupport/impl/RawDataReadSupport.java | 2 +-
.../hadoop/util/CarbonInputFormatUtil.java | 2 +-
.../carbondata/hadoop/util/SchemaReader.java | 2 +-
.../hadoop/ft/CarbonInputMapperTest.java | 2 +-
.../hadoop/test/util/StoreCreator.java | 14 +-
.../complexType/TestCreateTableWithDouble.scala | 2 +-
.../DataCompactionBoundaryConditionsTest.scala | 2 +-
.../DataCompactionCardinalityBoundryTest.scala | 3 +-
.../datacompaction/DataCompactionLockTest.scala | 5 +-
.../MajorCompactionIgnoreInMinorTest.scala | 3 +-
.../MajorCompactionStopsAfterCompaction.scala | 3 +-
.../dataretention/DataRetentionTestCase.scala | 5 +-
.../carbondata/spark/load/CarbonLoaderUtil.java | 14 +-
.../spark/load/DeleteLoadFolders.java | 2 +-
.../spark/merger/CarbonCompactionUtil.java | 2 +-
.../spark/merger/CarbonDataMergerUtil.java | 10 +-
.../spark/merger/RowResultMerger.java | 2 +-
.../carbondata/spark/merger/TableMeta.java | 2 +-
.../org/apache/carbondata/api/CarbonStore.scala | 3 +-
.../carbondata/spark/CarbonSparkFactory.scala | 2 +-
.../spark/DictionaryDetailHelper.scala | 2 +-
.../spark/rdd/CarbonGlobalDictionaryRDD.scala | 5 +-
.../spark/rdd/CarbonIUDMergerRDD.scala | 3 +-
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 14 +-
.../carbondata/spark/rdd/CarbonScanRDD.scala | 2 +-
.../spark/rdd/DataManagementFunc.scala | 7 +-
.../spark/util/GlobalDictionaryUtil.scala | 4 +-
.../readsupport/SparkRowReadSupportImpl.java | 2 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 23 +--
.../carbondata/spark/util/QueryPlanUtil.scala | 2 +-
.../sql/CarbonDatasourceHadoopRelation.scala | 2 +-
.../spark/sql/CarbonDictionaryDecoder.scala | 4 +-
.../sql/execution/command/IUDCommands.scala | 4 +-
.../execution/command/carbonTableSchema.scala | 7 +-
.../apache/spark/sql/hive/CarbonMetastore.scala | 5 +-
.../CompactionSystemLockFeatureTest.scala | 2 +-
.../DataCompactionMinorThresholdTest.scala | 2 +-
.../DataCompactionNoDictionaryTest.scala | 2 +-
.../datacompaction/DataCompactionTest.scala | 6 +-
.../spark/util/AllDictionaryTestCase.scala | 3 +-
.../AutoHighCardinalityIdentifyTestCase.scala | 5 +-
.../spark/util/DictionaryTestCaseUtil.scala | 2 +-
.../util/ExternalColumnDictionaryTestCase.scala | 3 +-
...GlobalDictionaryUtilConcurrentTestCase.scala | 3 +-
.../util/GlobalDictionaryUtilTestCase.scala | 3 +-
.../readsupport/SparkRowReadSupportImpl.java | 2 +-
.../carbondata/spark/CarbonSparkFactory.scala | 10 +-
.../spark/DictionaryDetailHelper.scala | 2 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 22 +--
.../carbondata/spark/util/QueryPlanUtil.scala | 2 +-
.../sql/CarbonDatasourceHadoopRelation.scala | 10 +-
.../spark/sql/CarbonDictionaryDecoder.scala | 2 +-
.../execution/command/carbonTableSchema.scala | 23 +--
.../apache/spark/sql/hive/CarbonMetastore.scala | 5 +-
.../processing/csvload/DataGraphExecuter.java | 2 +-
.../processing/csvload/GraphExecutionUtil.java | 4 +-
.../processing/datatypes/PrimitiveDataType.java | 2 +-
.../graphgenerator/GraphGenerator.java | 2 +-
.../processing/model/CarbonDataLoadSchema.java | 184 +++++++++++++++++
.../processing/model/CarbonLoadModel.java | 1 -
.../newflow/CarbonDataLoadConfiguration.java | 2 +-
.../processing/newflow/DataLoadExecutor.java | 2 +-
.../newflow/DataLoadProcessBuilder.java | 2 +-
.../impl/DictionaryFieldConverterImpl.java | 2 +-
.../converter/impl/FieldEncoderFactory.java | 2 +-
.../steps/DataConverterProcessorStepImpl.java | 2 +-
...ConverterProcessorWithBucketingStepImpl.java | 2 +-
.../steps/DataWriterProcessorStepImpl.java | 2 +-
.../sortandgroupby/sortdata/SortParameters.java | 2 +-
.../store/CarbonDataWriterFactory.java | 2 +-
.../store/CarbonFactDataHandlerColumnar.java | 2 +-
.../store/CarbonFactDataHandlerModel.java | 2 +-
.../writer/v2/CarbonFactDataWriterImplV2.java | 2 +-
.../FileStoreSurrogateKeyGenForCSV.java | 4 +-
.../util/CarbonDataProcessorUtil.java | 2 +-
.../processing/util/CarbonSchemaParser.java | 4 +-
.../carbon/datastore/BlockIndexStoreTest.java | 6 +-
.../carbondata/lcm/locks/LocalFileLockTest.java | 65 -------
.../lcm/locks/ZooKeeperLockingTest.java | 144 --------------
.../carbondata/test/util/StoreCreator.java | 14 +-
196 files changed, 2033 insertions(+), 2185 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/AbsoluteTableIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/AbsoluteTableIdentifier.java b/core/src/main/java/org/apache/carbondata/core/AbsoluteTableIdentifier.java
deleted file mode 100644
index e5823b9..0000000
--- a/core/src/main/java/org/apache/carbondata/core/AbsoluteTableIdentifier.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core;
-
-import java.io.File;
-import java.io.Serializable;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
-import org.apache.carbondata.core.util.CarbonUtil;
-
-/**
- * identifier which will have store path and carbon table identifier
- */
-public class AbsoluteTableIdentifier implements Serializable {
-
- /**
- * serializable version
- */
- private static final long serialVersionUID = 4695047103484427506L;
-
- /**
- * path of the store
- */
- private String storePath;
-
- /**
- * carbon table identifier which will have table name and table database
- * name
- */
- private CarbonTableIdentifier carbonTableIdentifier;
-
- public AbsoluteTableIdentifier(String storePath, CarbonTableIdentifier carbonTableIdentifier) {
- //TODO this should be moved to common place where path handling will be handled
- this.storePath = FileFactory.getUpdatedFilePath(storePath);
- this.carbonTableIdentifier = carbonTableIdentifier;
- }
-
- /**
- * @return the storePath
- */
- public String getStorePath() {
- return storePath;
- }
-
- /**
- * @return the carbonTableIdentifier
- */
- public CarbonTableIdentifier getCarbonTableIdentifier() {
- return carbonTableIdentifier;
- }
-
- public static AbsoluteTableIdentifier from(String dbName, String tableName) {
- CarbonTableIdentifier identifier = new CarbonTableIdentifier(dbName, tableName, "");
- return new AbsoluteTableIdentifier(CarbonUtil.getCarbonStorePath(), identifier);
- }
-
- public static AbsoluteTableIdentifier fromTablePath(String tablePath) {
- String formattedTablePath = tablePath.replace('\\', '/');
- String[] names = formattedTablePath.split("/");
- if (names.length < 3) {
- throw new IllegalArgumentException("invalid table path: " + tablePath);
- }
-
- String tableName = names[names.length - 1];
- String dbName = names[names.length - 2];
- String storePath = formattedTablePath.substring(0, formattedTablePath.lastIndexOf(dbName +
- CarbonCommonConstants.FILE_SEPARATOR + tableName));
-
- CarbonTableIdentifier identifier =
- new CarbonTableIdentifier(dbName, tableName, Long.toString(System.currentTimeMillis()));
- return new AbsoluteTableIdentifier(storePath, identifier);
- }
-
- public String getTablePath() {
- return getStorePath() + File.separator + getCarbonTableIdentifier().getDatabaseName() +
- File.separator + getCarbonTableIdentifier().getTableName();
- }
-
- /**
- * to get the hash code
- */
- @Override public int hashCode() {
- final int prime = 31;
- int result = 1;
- result =
- prime * result + ((carbonTableIdentifier == null) ? 0 : carbonTableIdentifier.hashCode());
- result = prime * result + ((storePath == null) ? 0 : storePath.hashCode());
- return result;
- }
-
- /**
- * to check this class is equal to
- * other object passed
- *
- * @param obj other object
- */
- @Override public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (!(obj instanceof AbsoluteTableIdentifier)) {
- return false;
- }
- AbsoluteTableIdentifier other = (AbsoluteTableIdentifier) obj;
- if (carbonTableIdentifier == null) {
- if (other.carbonTableIdentifier != null) {
- return false;
- }
- } else if (!carbonTableIdentifier.equals(other.carbonTableIdentifier)) {
- return false;
- }
- if (storePath == null) {
- if (other.storePath != null) {
- return false;
- }
- } else if (!storePath.equals(other.storePath)) {
- return false;
- }
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/CarbonDataLoadSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/CarbonDataLoadSchema.java b/core/src/main/java/org/apache/carbondata/core/CarbonDataLoadSchema.java
deleted file mode 100644
index 804cdf7..0000000
--- a/core/src/main/java/org/apache/carbondata/core/CarbonDataLoadSchema.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-
-/**
- * Wrapper Data Load Schema object which will be used to
- * support relation while data loading
- */
-public class CarbonDataLoadSchema implements Serializable {
-
- /**
- * default serializer
- */
- private static final long serialVersionUID = 1L;
-
- /**
- * CarbonTable info
- */
- private CarbonTable carbonTable;
-
- /**
- * dimension table and relation info
- */
- private List<DimensionRelation> dimensionRelationList;
-
- /**
- * CarbonDataLoadSchema constructor which takes CarbonTable
- *
- * @param carbonTable
- */
- public CarbonDataLoadSchema(CarbonTable carbonTable) {
- this.carbonTable = carbonTable;
- this.dimensionRelationList = new ArrayList<DimensionRelation>();
- }
-
- /**
- * get dimension relation list
- *
- * @return dimensionRelationList
- */
- public List<DimensionRelation> getDimensionRelationList() {
- return dimensionRelationList;
- }
-
- /**
- * get carbontable
- *
- * @return carbonTable
- */
- public CarbonTable getCarbonTable() {
- return carbonTable;
- }
-
- /**
- * Dimension Relation object which will be filled from
- * Load DML Command to support normalized table data load
- */
- public static class DimensionRelation implements Serializable {
- /**
- * default serializer
- */
- private static final long serialVersionUID = 1L;
-
- /**
- * dimension tableName
- */
- private String tableName;
-
- /**
- * relation with fact and dimension table
- */
- private Relation relation;
-
- /**
- * Columns to selected from dimension table.
- * Hierarchy in-memory table should be prepared
- * based on selected columns
- */
- private List<String> columns;
-
- /**
- * constructor
- *
- * @param tableName - dimension table name
- * @param relation - fact foreign key with dimension primary key mapping
- * @param columns - list of columns to be used from this dimension table
- */
- public DimensionRelation(String tableName, Relation relation,
- List<String> columns) {
- this.tableName = tableName;
- this.relation = relation;
- this.columns = columns;
- }
-
- /**
- * @return tableName
- */
- public String getTableName() {
- return tableName;
- }
-
- /**
- * @return relation
- */
- public Relation getRelation() {
- return relation;
- }
-
- /**
- * @return columns
- */
- public List<String> getColumns() {
- return columns;
- }
- }
-
- /**
- * Relation class to specify fact foreignkey column with
- * dimension primary key column
- */
- public static class Relation implements Serializable {
- /**
- * default serializer
- */
- private static final long serialVersionUID = 1L;
-
- /**
- * Fact foreign key column
- */
- private String factForeignKeyColumn;
-
- /**
- * dimension primary key column
- */
- private String dimensionPrimaryKeyColumn;
-
- /**
- * constructor
- *
- * @param factForeignKeyColumn - Fact Table Foreign key
- * @param dimensionPrimaryKeyColumn - Dimension Table primary key
- */
- public Relation(String factForeignKeyColumn, String dimensionPrimaryKeyColumn) {
- this.factForeignKeyColumn = factForeignKeyColumn;
- this.dimensionPrimaryKeyColumn = dimensionPrimaryKeyColumn;
- }
-
- /**
- * @return factForeignKeyColumn
- */
- public String getFactForeignKeyColumn() {
- return factForeignKeyColumn;
- }
-
- /**
- * @return dimensionPrimaryKeyColumn
- */
- public String getDimensionPrimaryKeyColumn() {
- return dimensionPrimaryKeyColumn;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/CarbonTableIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/CarbonTableIdentifier.java b/core/src/main/java/org/apache/carbondata/core/CarbonTableIdentifier.java
deleted file mode 100644
index 2789f64..0000000
--- a/core/src/main/java/org/apache/carbondata/core/CarbonTableIdentifier.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core;
-
-import java.io.File;
-import java.io.Serializable;
-
-/**
- * Identifier class which will hold the table qualified name
- */
-public class CarbonTableIdentifier implements Serializable {
-
- /**
- * database name
- */
- private String databaseName;
-
- /**
- * table name
- */
- private String tableName;
-
- /**
- * table id
- */
- private String tableId;
-
- /**
- * constructor
- */
- public CarbonTableIdentifier(String databaseName, String tableName, String tableId) {
- this.databaseName = databaseName;
- this.tableName = tableName;
- this.tableId = tableId;
- }
-
- /**
- * return database name
- */
- public String getDatabaseName() {
- return databaseName;
- }
-
- /**
- * return table name
- */
- public String getTableName() {
- return tableName;
- }
-
- /**
- * @return tableId
- */
- public String getTableId() {
- return tableId;
- }
-
- /**
- * @return table unique name
- */
- public String getTableUniqueName() {
- return databaseName + '_' + tableName;
- }
-
- /**
- *Creates the key for bad record lgger.
- */
- public String getBadRecordLoggerKey() {
- return databaseName + File.separator + tableName + '_' + tableName;
- }
-
- @Override public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((databaseName == null) ? 0 : databaseName.hashCode());
- result = prime * result + ((tableId == null) ? 0 : tableId.hashCode());
- result = prime * result + ((tableName == null) ? 0 : tableName.hashCode());
- return result;
- }
-
- @Override public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- CarbonTableIdentifier other = (CarbonTableIdentifier) obj;
- if (databaseName == null) {
- if (other.databaseName != null) {
- return false;
- }
- } else if (!databaseName.equals(other.databaseName)) {
- return false;
- }
- if (tableId == null) {
- if (other.tableId != null) {
- return false;
- }
- } else if (!tableId.equals(other.tableId)) {
- return false;
- }
- if (tableName == null) {
- if (other.tableName != null) {
- return false;
- }
- } else if (!tableName.equals(other.tableName)) {
- return false;
- }
- return true;
- }
-
- /*
- * @return table unidque name
- */
- @Override public String toString() {
- return databaseName + '_' + tableName;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/ColumnIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/ColumnIdentifier.java b/core/src/main/java/org/apache/carbondata/core/ColumnIdentifier.java
deleted file mode 100644
index 4fcbca6..0000000
--- a/core/src/main/java/org/apache/carbondata/core/ColumnIdentifier.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core;
-
-import java.io.Serializable;
-import java.util.Map;
-
-import org.apache.carbondata.core.metadata.DataType;
-
-/**
- * Column unique identifier
- */
-public class ColumnIdentifier implements Serializable {
-
- /**
- *
- */
- private static final long serialVersionUID = 1L;
-
- /**
- * column id
- */
- private String columnId;
-
- /**
- * column properties
- */
- private Map<String, String> columnProperties;
-
- private DataType dataType;
-
- /**
- * @param columnId
- * @param columnProperties
- */
- public ColumnIdentifier(String columnId, Map<String, String> columnProperties,
- DataType dataType) {
- this.columnId = columnId;
- this.columnProperties = columnProperties;
- this.dataType = dataType;
- }
-
- /**
- * @return columnId
- */
- public String getColumnId() {
- return columnId;
- }
-
- /**
- * @param columnProperty
- * @return
- */
- public String getColumnProperty(String columnProperty) {
- String property = null;
- if (null != columnProperties) {
- property = columnProperties.get(columnProperty);
- }
- return property;
- }
-
- public DataType getDataType() {
- return this.dataType;
- }
-
- @Override public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((columnId == null) ? 0 : columnId.hashCode());
- return result;
- }
-
- @Override public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- ColumnIdentifier other = (ColumnIdentifier) obj;
- if (columnId == null) {
- if (other.columnId != null) {
- return false;
- }
- } else if (!columnId.equals(other.columnId)) {
- return false;
- }
- return true;
- }
-
- @Override public String toString() {
- return "ColumnIdentifier [columnId=" + columnId + "]";
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/ColumnarFormatVersion.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/ColumnarFormatVersion.java b/core/src/main/java/org/apache/carbondata/core/ColumnarFormatVersion.java
deleted file mode 100644
index 8fe03b4..0000000
--- a/core/src/main/java/org/apache/carbondata/core/ColumnarFormatVersion.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core;
-
-public enum ColumnarFormatVersion {
- V1((short)1),
- V2((short)2);
-
- private short version;
- ColumnarFormatVersion(short version) {
- this.version = version;
- }
-
- @Override
- public String toString() {
- return "ColumnarFormatV" + version;
- }
-
- public short number() {
- return version;
- }
-
- public static ColumnarFormatVersion valueOf(short version) {
- switch (version) {
- case 0:
- // before multiple reader support, for existing carbon file, it is version 1
- return V1;
- case 1:
- // after multiple reader support, user can write new file with version 1
- return V1;
- default:
- return V2;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/atomic/AtomicFileOperations.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/atomic/AtomicFileOperations.java b/core/src/main/java/org/apache/carbondata/core/atomic/AtomicFileOperations.java
new file mode 100644
index 0000000..1061723
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/atomic/AtomicFileOperations.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.atomic;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+public interface AtomicFileOperations {
+
+ DataInputStream openForRead() throws IOException;
+
+ void close() throws IOException;
+
+ DataOutputStream openForWrite(FileWriteOperation operation) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/atomic/AtomicFileOperationsImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/atomic/AtomicFileOperationsImpl.java b/core/src/main/java/org/apache/carbondata/core/atomic/AtomicFileOperationsImpl.java
new file mode 100644
index 0000000..33eeae0
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/atomic/AtomicFileOperationsImpl.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.atomic;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.impl.FileFactory.FileType;
+
+public class AtomicFileOperationsImpl implements AtomicFileOperations {
+
+ private String filePath;
+
+ private FileType fileType;
+
+ private String tempWriteFilePath;
+
+ private DataOutputStream dataOutStream;
+
+ public AtomicFileOperationsImpl(String filePath, FileType fileType) {
+ this.filePath = filePath;
+
+ this.fileType = fileType;
+ }
+
+ @Override public DataInputStream openForRead() throws IOException {
+ return FileFactory.getDataInputStream(filePath, fileType);
+ }
+
+ @Override public DataOutputStream openForWrite(FileWriteOperation operation) throws IOException {
+
+ filePath = filePath.replace("\\", "/");
+
+ tempWriteFilePath = filePath + CarbonCommonConstants.TEMPWRITEFILEEXTENSION;
+
+ if (FileFactory.isFileExist(tempWriteFilePath, fileType)) {
+ FileFactory.getCarbonFile(tempWriteFilePath, fileType).delete();
+ }
+
+ FileFactory.createNewFile(tempWriteFilePath, fileType);
+
+ dataOutStream = FileFactory.getDataOutputStream(tempWriteFilePath, fileType);
+
+ return dataOutStream;
+
+ }
+
+ @Override public void close() throws IOException {
+
+ if (null != dataOutStream) {
+ dataOutStream.close();
+
+ CarbonFile tempFile = FileFactory.getCarbonFile(tempWriteFilePath, fileType);
+
+ if (!tempFile.renameForce(filePath)) {
+ throw new IOException("temporary file renaming failed, src="
+ + tempFile.getPath() + ", dest=" + filePath);
+ }
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/atomic/FileWriteOperation.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/atomic/FileWriteOperation.java b/core/src/main/java/org/apache/carbondata/core/atomic/FileWriteOperation.java
new file mode 100644
index 0000000..535857f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/atomic/FileWriteOperation.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.atomic;
+
+public enum FileWriteOperation {
+
+ APPEND, OVERWRITE
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoader.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoader.java
index 699a9fb..59a93f3 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoader.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoader.java
@@ -21,7 +21,7 @@ package org.apache.carbondata.core.cache.dictionary;
import java.io.IOException;
-import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
public interface DictionaryCacheLoader {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
index 7d57676..c60b911 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
@@ -25,8 +25,8 @@ import java.util.Iterator;
import java.util.List;
import org.apache.carbondata.core.service.CarbonCommonFactory;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.reader.CarbonDictionaryReader;
import org.apache.carbondata.core.reader.sortindex.CarbonDictionarySortIndexReader;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java
index 8c48b52..010fabe 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifier.java
@@ -19,8 +19,8 @@
package org.apache.carbondata.core.cache.dictionary;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.metadata.DataType;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java
index 68d0e95..8fe4601 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java
@@ -35,8 +35,8 @@ import java.util.concurrent.TimeUnit;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.CarbonLRUCache;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.block.BlockIndex;
import org.apache.carbondata.core.datastore.block.BlockInfo;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
index 1ae8d28..8feae0c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
@@ -32,7 +32,7 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.Cache;
import org.apache.carbondata.core.cache.CarbonLRUCache;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.block.SegmentTaskIndex;
import org.apache.carbondata.core.datastore.block.SegmentTaskIndexWrapper;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/datastore/TableSegmentUniqueIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/TableSegmentUniqueIdentifier.java b/core/src/main/java/org/apache/carbondata/core/datastore/TableSegmentUniqueIdentifier.java
index 23db251..71b7c3f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/TableSegmentUniqueIdentifier.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/TableSegmentUniqueIdentifier.java
@@ -22,8 +22,8 @@ package org.apache.carbondata.core.datastore;
import java.util.List;
import java.util.Map;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
index a5061d2..4f20ceb 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
@@ -22,7 +22,7 @@ import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
-import org.apache.carbondata.core.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.path.CarbonTablePath.DataFileUtil;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockUniqueIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockUniqueIdentifier.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockUniqueIdentifier.java
index e847227..f2fbba7 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockUniqueIdentifier.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockUniqueIdentifier.java
@@ -1,7 +1,7 @@
package org.apache.carbondata.core.datastore.block;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
index af7034c..5ac5702 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
@@ -18,7 +18,7 @@
*/
package org.apache.carbondata.core.datastore.chunk.reader;
-import org.apache.carbondata.core.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.datastore.chunk.reader.dimension.v1.CompressedDimensionChunkFileBasedReaderV1;
import org.apache.carbondata.core.datastore.chunk.reader.dimension.v2.CompressedDimensionChunkFileBasedReaderV2;
import org.apache.carbondata.core.datastore.chunk.reader.measure.v1.CompressedMeasureChunkFileBasedReaderV1;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java
index 9df57b0..b95f2c9 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java
@@ -30,8 +30,8 @@ import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.cache.CacheType;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java b/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
new file mode 100644
index 0000000..59aaf53
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.metadata;
+
+import java.io.File;
+import java.io.Serializable;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+/**
+ * identifier which will have store path and carbon table identifier
+ */
+public class AbsoluteTableIdentifier implements Serializable {
+
+ /**
+ * serializable version
+ */
+ private static final long serialVersionUID = 4695047103484427506L;
+
+ /**
+ * path of the store
+ */
+ private String storePath;
+
+ /**
+ * carbon table identifier which will have table name and table database name
+ */
+ private CarbonTableIdentifier carbonTableIdentifier;
+
+ public AbsoluteTableIdentifier(String storePath, CarbonTableIdentifier carbonTableIdentifier) {
+ //TODO this should be moved to common place where path handling will be handled
+ this.storePath = FileFactory.getUpdatedFilePath(storePath);
+ this.carbonTableIdentifier = carbonTableIdentifier;
+ }
+
+ /**
+ * @return the storePath
+ */
+ public String getStorePath() {
+ return storePath;
+ }
+
+ /**
+ * @return the carbonTableIdentifier
+ */
+ public CarbonTableIdentifier getCarbonTableIdentifier() {
+ return carbonTableIdentifier;
+ }
+
+ public static AbsoluteTableIdentifier from(String dbName, String tableName) {
+ CarbonTableIdentifier identifier = new CarbonTableIdentifier(dbName, tableName, "");
+ return new AbsoluteTableIdentifier(CarbonUtil.getCarbonStorePath(), identifier);
+ }
+
+ public static AbsoluteTableIdentifier fromTablePath(String tablePath) {
+ String formattedTablePath = tablePath.replace('\\', '/');
+ String[] names = formattedTablePath.split("/");
+ if (names.length < 3) {
+ throw new IllegalArgumentException("invalid table path: " + tablePath);
+ }
+
+ String tableName = names[names.length - 1];
+ String dbName = names[names.length - 2];
+ String storePath = formattedTablePath.substring(0, formattedTablePath.lastIndexOf(dbName +
+ CarbonCommonConstants.FILE_SEPARATOR + tableName));
+
+ CarbonTableIdentifier identifier =
+ new CarbonTableIdentifier(dbName, tableName, Long.toString(System.currentTimeMillis()));
+ return new AbsoluteTableIdentifier(storePath, identifier);
+ }
+
+ public String getTablePath() {
+ return getStorePath() + File.separator + getCarbonTableIdentifier().getDatabaseName() +
+ File.separator + getCarbonTableIdentifier().getTableName();
+ }
+
+ /**
+ * to get the hash code
+ */
+ @Override public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result =
+ prime * result + ((carbonTableIdentifier == null) ? 0 : carbonTableIdentifier.hashCode());
+ result = prime * result + ((storePath == null) ? 0 : storePath.hashCode());
+ return result;
+ }
+
+ /**
+ * to check this class is equal to
+ * other object passed
+ *
+ * @param obj other object
+ */
+ @Override public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (!(obj instanceof AbsoluteTableIdentifier)) {
+ return false;
+ }
+ AbsoluteTableIdentifier other = (AbsoluteTableIdentifier) obj;
+ if (carbonTableIdentifier == null) {
+ if (other.carbonTableIdentifier != null) {
+ return false;
+ }
+ } else if (!carbonTableIdentifier.equals(other.carbonTableIdentifier)) {
+ return false;
+ }
+ if (storePath == null) {
+ if (other.storePath != null) {
+ return false;
+ }
+ } else if (!storePath.equals(other.storePath)) {
+ return false;
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/metadata/CarbonTableIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/CarbonTableIdentifier.java b/core/src/main/java/org/apache/carbondata/core/metadata/CarbonTableIdentifier.java
new file mode 100644
index 0000000..7d2fb18
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/CarbonTableIdentifier.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.metadata;
+
+import java.io.File;
+import java.io.Serializable;
+
+/**
+ * Identifier class which will hold the table qualified name
+ */
+public class CarbonTableIdentifier implements Serializable {
+
+ /**
+ * database name
+ */
+ private String databaseName;
+
+ /**
+ * table name
+ */
+ private String tableName;
+
+ /**
+ * table id
+ */
+ private String tableId;
+
+ /**
+ * constructor
+ */
+ public CarbonTableIdentifier(String databaseName, String tableName, String tableId) {
+ this.databaseName = databaseName;
+ this.tableName = tableName;
+ this.tableId = tableId;
+ }
+
+ /**
+ * return database name
+ */
+ public String getDatabaseName() {
+ return databaseName;
+ }
+
+ /**
+ * return table name
+ */
+ public String getTableName() {
+ return tableName;
+ }
+
+ /**
+ * @return tableId
+ */
+ public String getTableId() {
+ return tableId;
+ }
+
+ /**
+ * @return table unique name
+ */
+ public String getTableUniqueName() {
+ return databaseName + '_' + tableName;
+ }
+
+ /**
+ *Creates the key for bad record lgger.
+ */
+ public String getBadRecordLoggerKey() {
+ return databaseName + File.separator + tableName + '_' + tableName;
+ }
+
+ @Override public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((databaseName == null) ? 0 : databaseName.hashCode());
+ result = prime * result + ((tableId == null) ? 0 : tableId.hashCode());
+ result = prime * result + ((tableName == null) ? 0 : tableName.hashCode());
+ return result;
+ }
+
+ @Override public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ CarbonTableIdentifier other = (CarbonTableIdentifier) obj;
+ if (databaseName == null) {
+ if (other.databaseName != null) {
+ return false;
+ }
+ } else if (!databaseName.equals(other.databaseName)) {
+ return false;
+ }
+ if (tableId == null) {
+ if (other.tableId != null) {
+ return false;
+ }
+ } else if (!tableId.equals(other.tableId)) {
+ return false;
+ }
+ if (tableName == null) {
+ if (other.tableName != null) {
+ return false;
+ }
+ } else if (!tableName.equals(other.tableName)) {
+ return false;
+ }
+ return true;
+ }
+
+ /*
+ * @return table unidque name
+ */
+ @Override public String toString() {
+ return databaseName + '_' + tableName;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/metadata/ColumnarFormatVersion.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/ColumnarFormatVersion.java b/core/src/main/java/org/apache/carbondata/core/metadata/ColumnarFormatVersion.java
new file mode 100644
index 0000000..fb15cc5
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/ColumnarFormatVersion.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.metadata;
+
+public enum ColumnarFormatVersion {
+ V1((short)1),
+ V2((short)2);
+
+ private short version;
+ ColumnarFormatVersion(short version) {
+ this.version = version;
+ }
+
+ @Override
+ public String toString() {
+ return "ColumnarFormatV" + version;
+ }
+
+ public short number() {
+ return version;
+ }
+
+ public static ColumnarFormatVersion valueOf(short version) {
+ switch (version) {
+ case 0:
+ // before multiple reader support, for existing carbon file, it is version 1
+ return V1;
+ case 1:
+ // after multiple reader support, user can write new file with version 1
+ return V1;
+ default:
+ return V2;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/DataFileFooter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/DataFileFooter.java b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/DataFileFooter.java
index 807b039..d064ab3 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/DataFileFooter.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/DataFileFooter.java
@@ -21,7 +21,7 @@ package org.apache.carbondata.core.metadata.blocklet;
import java.io.Serializable;
import java.util.List;
-import org.apache.carbondata.core.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.datastore.block.BlockInfo;
import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index b4c9f71..dae3c26 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -24,8 +24,8 @@ import java.util.*;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.metadata.schema.BucketingInfo;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java
index 50c800e..511d51e 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/CarbonColumn.java
@@ -23,7 +23,7 @@ import java.io.Serializable;
import java.util.List;
import java.util.Map;
-import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.metadata.Encoding;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/path/CarbonStorePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/path/CarbonStorePath.java b/core/src/main/java/org/apache/carbondata/core/path/CarbonStorePath.java
index 833d5eb..4f95fd4 100644
--- a/core/src/main/java/org/apache/carbondata/core/path/CarbonStorePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/path/CarbonStorePath.java
@@ -20,8 +20,8 @@ package org.apache.carbondata.core.path;
import java.io.File;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.hadoop.fs.Path;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/path/CarbonTablePath.java
index b762ca1..45c947f 100644
--- a/core/src/main/java/org/apache/carbondata/core/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/path/CarbonTablePath.java
@@ -20,7 +20,7 @@ package org.apache.carbondata.core.path;
import java.io.File;
-import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java
index b76353c..097c25b 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.update.DeleteDeltaBlockDetails;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.fileoperations.AtomicFileOperations;
-import org.apache.carbondata.fileoperations.AtomicFileOperationsImpl;
+import org.apache.carbondata.core.atomic.AtomicFileOperations;
+import org.apache.carbondata.core.atomic.AtomicFileOperationsImpl;
import com.google.gson.Gson;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
index 43ea57f..318836c 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
@@ -24,8 +24,8 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.carbondata.core.service.CarbonCommonFactory;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.service.PathService;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java
index 27b1fc3..76b22cb 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java
@@ -27,8 +27,8 @@ import java.util.List;
import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.core.cache.dictionary.ColumnDictionaryChunkIterator;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.service.PathService;
import org.apache.carbondata.format.ColumnDictionaryChunk;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
index 1dd51d4..1b95507 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
@@ -24,8 +24,8 @@ import java.util.List;
import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index 05df636..d2da9d3 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -33,7 +33,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.common.logging.impl.StandardLogService;
import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.cache.CacheType;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.BlockIndexStore;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
index 7b29c8b..6f4a37e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
@@ -21,7 +21,7 @@ package org.apache.carbondata.core.scan.executor.infos;
import java.util.Map;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
index 676d978..a9c6df0 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
@@ -37,8 +37,8 @@ import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.cache.CacheType;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
index 0b82b53..31321ec 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
@@ -25,7 +25,7 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.datastore.DataRefNodeFinder;
import org.apache.carbondata.core.datastore.IndexKey;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java
index 4199429..038f496 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java
@@ -22,7 +22,7 @@ package org.apache.carbondata.core.scan.filter;
import java.io.IOException;
import java.util.List;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.scan.expression.Expression;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index ab0c7e8..9af8d53 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -46,7 +46,7 @@ import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryChunksWrapper;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
import org.apache.carbondata.core.cache.dictionary.ForwardDictionary;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.metadata.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index 0c1b1a1..a47704d 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -31,7 +31,7 @@ import java.util.Map;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
import org.apache.carbondata.core.metadata.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
index 11ec991..dfab0b1 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
@@ -22,7 +22,7 @@ import java.io.IOException;
import java.util.BitSet;
import java.util.List;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
index fb7c86d..825f59e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
@@ -22,7 +22,7 @@ import java.io.IOException;
import java.util.BitSet;
import java.util.List;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index 33c2e50..1eb47e6 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -22,7 +22,7 @@ import java.io.IOException;
import java.util.BitSet;
import java.util.List;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
index 6f2d3e2..83fa050 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
@@ -22,7 +22,7 @@ import java.io.IOException;
import java.util.BitSet;
import java.util.List;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
index ce18b75..69cf216 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
@@ -22,7 +22,7 @@ import java.io.IOException;
import java.util.List;
import java.util.SortedMap;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.metadata.Encoding;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java
index b395e7c..9b42ecc 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java
@@ -23,7 +23,7 @@ import java.io.Serializable;
import java.util.List;
import java.util.SortedMap;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java
index 4ffb8ba..11f9eaa 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java
@@ -22,7 +22,7 @@ package org.apache.carbondata.core.scan.filter.resolver;
import java.util.List;
import java.util.SortedMap;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.scan.expression.BinaryExpression;
import org.apache.carbondata.core.scan.expression.Expression;
[55/57] [abbrv] incubator-carbondata git commit: move package in core
Posted by ja...@apache.org.
move package in core
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/f75ddfb6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/f75ddfb6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/f75ddfb6
Branch: refs/heads/movecorepackage
Commit: f75ddfb6c91f3a624adab5abd2acdb28340ebbf9
Parents: 4831619 392bc29
Author: jackylk <ja...@huawei.com>
Authored: Sat Jan 14 15:42:07 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Sat Jan 14 15:42:07 2017 +0800
----------------------------------------------------------------------
.../dictionary/DoubleArrayTrieDictionary.java | 384 +++++++++++++++++++
.../core/constants/CarbonCommonConstants.java | 10 +
.../chunk/store/MeasureChunkStoreFactory.java | 2 +-
.../executor/impl/AbstractQueryExecutor.java | 10 +
.../scan/executor/impl/DetailQueryExecutor.java | 3 +-
.../impl/VectorDetailQueryExecutor.java | 3 +-
.../processor/AbstractDataBlockIterator.java | 10 +-
.../processor/impl/DataBlockIteratorImpl.java | 6 +-
.../core/scan/result/AbstractScannedResult.java | 1 -
.../AbstractDetailQueryResultIterator.java | 16 +-
.../core/scan/scanner/impl/FilterScanner.java | 2 +
.../carbondata/core/util/CarbonProperties.java | 3 +-
integration/spark-common-test/pom.xml | 6 +-
...tCreateTableWithDatabaseNameCaseChange.scala | 60 +++
.../DataCompactionBlockletBoundryTest.scala | 2 +-
.../spark/rdd/CarbonDataLoadRDD.scala | 14 +-
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 5 +-
.../spark/rdd/NewCarbonDataLoadRDD.scala | 30 +-
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 25 +-
.../spark/sql/CarbonCatalystOperators.scala | 7 +
.../org/apache/spark/sql/CarbonSqlParser.scala | 26 +-
.../spark/sql/hive/CarbonStrategies.scala | 2 +
.../spark/sql/CarbonDictionaryDecoder.scala | 3 +-
.../org/apache/spark/sql/CarbonSource.scala | 33 +-
.../sql/parser/CarbonSpark2SqlParser.scala | 17 +-
.../spark/sql/parser/CarbonSparkSqlParser.scala | 15 +-
.../processing/csvreaderstep/CsvInput.java | 11 +-
.../csvreaderstep/JavaRddIterator.java | 32 --
.../processing/csvreaderstep/RddInputUtils.java | 12 +-
.../processing/model/CarbonLoadModel.java | 15 +
.../newflow/CarbonDataLoadConfiguration.java | 10 +
.../newflow/DataLoadProcessBuilder.java | 1 +
.../sort/impl/ParallelReadMergeSorterImpl.java | 3 -
.../newflow/steps/InputProcessorStepImpl.java | 45 ++-
.../sortandgroupby/sortdata/SortDataRows.java | 7 +-
.../util/CarbonDataProcessorUtil.java | 30 --
.../carbon/datastore/BlockIndexStoreTest.java | 2 +-
37 files changed, 696 insertions(+), 167 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f75ddfb6/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/MeasureChunkStoreFactory.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/MeasureChunkStoreFactory.java
index a95ae11,0000000..cb2352f
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/MeasureChunkStoreFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/MeasureChunkStoreFactory.java
@@@ -1,97 -1,0 +1,97 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.datastore.chunk.store;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeByteMeasureChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeDoubleMeasureChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeIntMeasureChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeLongMeasureChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeShortMeasureChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeByteMeasureChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeDoubleMeasureChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeIntMeasureChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeLongMeasureChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeShortMeasureChunkStore;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+/**
+ * Factory class for getting the measure store type
+ */
+public class MeasureChunkStoreFactory {
+
+ /**
+ * instance type
+ */
+ public static final MeasureChunkStoreFactory INSTANCE = new MeasureChunkStoreFactory();
+
+ /**
+ * is unsafe
+ */
+ private static final boolean isUnsafe;
+
+ static {
+ isUnsafe = Boolean.parseBoolean(CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.ENABLE_UNSAFE_IN_QUERY_EXECUTION,
+ CarbonCommonConstants.ENABLE_UNSAFE_IN_QUERY_EXECUTION_DEFAULTVALUE));
+ }
+
+ private MeasureChunkStoreFactory() {
+ }
+
+ /**
+ * Below method will be used to get the measure data chunk store based on data type
+ *
+ * @param dataType data type
+ * @param numberOfRows number of rows
+ * @return measure chunk store
+ */
+ public MeasureDataChunkStore getMeasureDataChunkStore(DataType dataType, int numberOfRows) {
- if (isUnsafe) {
++ if (!isUnsafe) {
+ switch (dataType) {
+ case DATA_BYTE:
+ return new SafeByteMeasureChunkStore(numberOfRows);
+ case DATA_SHORT:
+ return new SafeShortMeasureChunkStore(numberOfRows);
+ case DATA_INT:
+ return new SafeIntMeasureChunkStore(numberOfRows);
+ case DATA_LONG:
+ return new SafeLongMeasureChunkStore(numberOfRows);
+ case DATA_DOUBLE:
+ return new SafeDoubleMeasureChunkStore(numberOfRows);
+ default:
+ return new SafeDoubleMeasureChunkStore(numberOfRows);
+ }
+ } else {
+ switch (dataType) {
+ case DATA_BYTE:
+ return new UnsafeByteMeasureChunkStore(numberOfRows);
+ case DATA_SHORT:
+ return new UnsafeShortMeasureChunkStore(numberOfRows);
+ case DATA_INT:
+ return new UnsafeIntMeasureChunkStore(numberOfRows);
+ case DATA_LONG:
+ return new UnsafeLongMeasureChunkStore(numberOfRows);
+ case DATA_DOUBLE:
+ return new UnsafeDoubleMeasureChunkStore(numberOfRows);
+ default:
+ return new UnsafeDoubleMeasureChunkStore(numberOfRows);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f75ddfb6/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index 9eedb75,0000000..03826e0
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@@ -1,483 -1,0 +1,493 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.executor.impl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
++import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.common.logging.impl.StandardLogService;
+import org.apache.carbondata.core.cache.CacheProvider;
+import org.apache.carbondata.core.cache.CacheType;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.BlockIndexStore;
+import org.apache.carbondata.core.datastore.IndexKey;
+import org.apache.carbondata.core.datastore.block.AbstractIndex;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.datastore.block.TableBlockUniqueIdentifier;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.keygenerator.KeyGenerator;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.metadata.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
+import org.apache.carbondata.core.path.CarbonStorePath;
+import org.apache.carbondata.core.scan.executor.QueryExecutor;
+import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
+import org.apache.carbondata.core.scan.executor.infos.AggregatorInfo;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.executor.util.QueryUtil;
+import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.model.QueryDimension;
+import org.apache.carbondata.core.scan.model.QueryMeasure;
+import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.core.stats.QueryStatistic;
+import org.apache.carbondata.core.stats.QueryStatisticsConstants;
+import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+import org.apache.commons.lang3.ArrayUtils;
+
+/**
+ * This class provides a skeletal implementation of the {@link QueryExecutor}
+ * interface to minimize the effort required to implement this interface. This
+ * will be used to prepare all the properties required for query execution
+ */
+public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(AbstractQueryExecutor.class.getName());
+ /**
+ * holder for query properties which will be used to execute the query
+ */
+ protected QueryExecutorProperties queryProperties;
+
++ /**
++ * query result iterator which will execute the query
++ * and give the result
++ */
++ protected CarbonIterator queryIterator;
++
+ public AbstractQueryExecutor() {
+ queryProperties = new QueryExecutorProperties();
+ }
+
+ /**
+ * Below method will be used to fill the executor properties based on query
+ * model it will parse the query model and get the detail and fill it in
+ * query properties
+ *
+ * @param queryModel
+ */
+ protected void initQuery(QueryModel queryModel) throws IOException {
+ StandardLogService.setThreadName(StandardLogService.getPartitionID(
+ queryModel.getAbsoluteTableIdentifier().getCarbonTableIdentifier().getTableName()),
+ queryModel.getQueryId());
+ LOGGER.info("Query will be executed on table: " + queryModel.getAbsoluteTableIdentifier()
+ .getCarbonTableIdentifier().getTableName());
+ // add executor service for query execution
+ queryProperties.executorService = Executors.newFixedThreadPool(1);
+ // Initializing statistics list to record the query statistics
+ // creating copy on write to handle concurrent scenario
+ queryProperties.queryStatisticsRecorder =
+ CarbonTimeStatisticsFactory.createExecutorRecorder(queryModel.getQueryId());
+ queryModel.setStatisticsRecorder(queryProperties.queryStatisticsRecorder);
+ QueryUtil.resolveQueryModel(queryModel);
+ QueryStatistic queryStatistic = new QueryStatistic();
+ // sort the block info
+ // so block will be loaded in sorted order this will be required for
+ // query execution
+ Collections.sort(queryModel.getTableBlockInfos());
+ // get the table blocks
+ CacheProvider cacheProvider = CacheProvider.getInstance();
+ BlockIndexStore<TableBlockUniqueIdentifier, AbstractIndex> cache =
+ (BlockIndexStore) cacheProvider
+ .createCache(CacheType.EXECUTOR_BTREE, queryModel.getTable().getStorePath());
+ // remove the invalid table blocks, block which is deleted or compacted
+ cache.removeTableBlocks(queryModel.getInvalidSegmentIds(),
+ queryModel.getAbsoluteTableIdentifier());
+ List<TableBlockUniqueIdentifier> tableBlockUniqueIdentifiers =
+ prepareTableBlockUniqueIdentifier(queryModel.getTableBlockInfos(),
+ queryModel.getAbsoluteTableIdentifier());
+ cache.removeTableBlocksIfHorizontalCompactionDone(queryModel);
+ queryProperties.dataBlocks = cache.getAll(tableBlockUniqueIdentifiers);
+ queryStatistic
+ .addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_EXECUTOR, System.currentTimeMillis());
+ queryProperties.queryStatisticsRecorder.recordStatistics(queryStatistic);
+ //
+ // // updating the restructuring infos for the query
+ queryProperties.keyStructureInfo = getKeyStructureInfo(queryModel,
+ queryProperties.dataBlocks.get(queryProperties.dataBlocks.size() - 1).getSegmentProperties()
+ .getDimensionKeyGenerator());
+
+ // calculating the total number of aggeragted columns
+ int aggTypeCount = queryModel.getQueryMeasures().size();
+
+ int currentIndex = 0;
+ DataType[] dataTypes = new DataType[aggTypeCount];
+
+ for (QueryMeasure carbonMeasure : queryModel.getQueryMeasures()) {
+ // adding the data type and aggregation type of all the measure this
+ // can be used
+ // to select the aggregator
+ dataTypes[currentIndex] = carbonMeasure.getMeasure().getDataType();
+ currentIndex++;
+ }
+ queryProperties.measureDataTypes = dataTypes;
+ // as aggregation will be executed in following order
+ // 1.aggregate dimension expression
+ // 2. expression
+ // 3. query measure
+ // so calculating the index of the expression start index
+ // and measure column start index
+ queryProperties.filterMeasures = new HashSet<>();
+ queryProperties.complexFilterDimension = new HashSet<>();
+ QueryUtil.getAllFilterDimensions(queryModel.getFilterExpressionResolverTree(),
+ queryProperties.complexFilterDimension, queryProperties.filterMeasures);
+
+ queryStatistic = new QueryStatistic();
+ // dictionary column unique column id to dictionary mapping
+ // which will be used to get column actual data
+ queryProperties.columnToDictionayMapping = QueryUtil
+ .getDimensionDictionaryDetail(queryModel.getQueryDimension(),
+ queryProperties.complexFilterDimension, queryModel.getAbsoluteTableIdentifier());
+ queryStatistic
+ .addStatistics(QueryStatisticsConstants.LOAD_DICTIONARY, System.currentTimeMillis());
+ queryProperties.queryStatisticsRecorder.recordStatistics(queryStatistic);
+ queryModel.setColumnToDictionaryMapping(queryProperties.columnToDictionayMapping);
+ }
+
+ private List<TableBlockUniqueIdentifier> prepareTableBlockUniqueIdentifier(
+ List<TableBlockInfo> tableBlockInfos, AbsoluteTableIdentifier absoluteTableIdentifier) {
+ List<TableBlockUniqueIdentifier> tableBlockUniqueIdentifiers =
+ new ArrayList<>(tableBlockInfos.size());
+ for (TableBlockInfo blockInfo : tableBlockInfos) {
+ tableBlockUniqueIdentifiers
+ .add(new TableBlockUniqueIdentifier(absoluteTableIdentifier, blockInfo));
+ }
+ return tableBlockUniqueIdentifiers;
+ }
+
+ /**
+ * Below method will be used to get the key structure info for the query
+ *
+ * @param queryModel query model
+ * @param keyGenerator
+ * @return key structure info
+ */
+ private KeyStructureInfo getKeyStructureInfo(QueryModel queryModel, KeyGenerator keyGenerator) {
+ // getting the masked byte range for dictionary column
+ int[] maskByteRanges =
+ QueryUtil.getMaskedByteRange(queryModel.getQueryDimension(), keyGenerator);
+
+ // max key for the dictionary dimension present in the query
+ byte[] maxKey = null;
+ try {
+ // getting the max key which will be used to masked and get the
+ // masked key
+ maxKey = QueryUtil.getMaxKeyBasedOnDimensions(queryModel.getQueryDimension(), keyGenerator);
+ } catch (KeyGenException e) {
+ LOGGER.error(e, "problem while getting the max key");
+ }
+
+ KeyStructureInfo restructureInfos = new KeyStructureInfo();
+ restructureInfos.setKeyGenerator(keyGenerator);
+ restructureInfos.setMaskByteRanges(maskByteRanges);
+ restructureInfos.setMaxKey(maxKey);
+ return restructureInfos;
+ }
+
+ protected List<BlockExecutionInfo> getBlockExecutionInfos(QueryModel queryModel)
+ throws IOException, QueryExecutionException {
+ initQuery(queryModel);
+ List<BlockExecutionInfo> blockExecutionInfoList = new ArrayList<BlockExecutionInfo>();
+ // fill all the block execution infos for all the blocks selected in
+ // query
+ // and query will be executed based on that infos
+ for (int i = 0; i < queryProperties.dataBlocks.size(); i++) {
+ blockExecutionInfoList.add(
+ getBlockExecutionInfoForBlock(queryModel, queryProperties.dataBlocks.get(i),
+ queryModel.getTableBlockInfos().get(i).getBlockletInfos().getStartBlockletNumber(),
+ queryModel.getTableBlockInfos().get(i).getBlockletInfos().getNumberOfBlockletToScan(),
+ queryModel.getTableBlockInfos().get(i).getFilePath()));
+ }
+ if (null != queryModel.getStatisticsRecorder()) {
+ QueryStatistic queryStatistic = new QueryStatistic();
+ queryStatistic.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKS_NUM,
+ blockExecutionInfoList.size());
+ queryModel.getStatisticsRecorder().recordStatistics(queryStatistic);
+ }
+ return blockExecutionInfoList;
+ }
+
+ /**
+ * Below method will be used to get the block execution info which is
+ * required to execute any block based on query model
+ *
+ * @param queryModel query model from user query
+ * @param blockIndex block index
+ * @return block execution info
+ * @throws QueryExecutionException any failure during block info creation
+ */
+ protected BlockExecutionInfo getBlockExecutionInfoForBlock(QueryModel queryModel,
+ AbstractIndex blockIndex, int startBlockletIndex, int numberOfBlockletToScan, String filePath)
+ throws QueryExecutionException {
+ BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
+ SegmentProperties segmentProperties = blockIndex.getSegmentProperties();
+ List<CarbonDimension> tableBlockDimensions = segmentProperties.getDimensions();
+ KeyGenerator blockKeyGenerator = segmentProperties.getDimensionKeyGenerator();
+
+ // below is to get only those dimension in query which is present in the
+ // table block
+ List<QueryDimension> updatedQueryDimension = RestructureUtil
+ .getUpdatedQueryDimension(queryModel.getQueryDimension(), tableBlockDimensions,
+ segmentProperties.getComplexDimensions());
+ // TODO add complex dimension children
+ int[] maskByteRangesForBlock =
+ QueryUtil.getMaskedByteRange(updatedQueryDimension, blockKeyGenerator);
+ int[] maksedByte =
+ QueryUtil.getMaskedByte(blockKeyGenerator.getKeySizeInBytes(), maskByteRangesForBlock);
+ int tableFactPathLength = CarbonStorePath
+ .getCarbonTablePath(queryModel.getAbsoluteTableIdentifier().getStorePath(),
+ queryModel.getAbsoluteTableIdentifier().getCarbonTableIdentifier()).getFactDir()
+ .length() + 1;
+ blockExecutionInfo.setBlockId(filePath.substring(tableFactPathLength));
+ blockExecutionInfo.setStartBlockletIndex(startBlockletIndex);
+ blockExecutionInfo.setNumberOfBlockletToScan(numberOfBlockletToScan);
+ blockExecutionInfo.setQueryDimensions(
+ updatedQueryDimension.toArray(new QueryDimension[updatedQueryDimension.size()]));
+ blockExecutionInfo.setQueryMeasures(queryModel.getQueryMeasures()
+ .toArray(new QueryMeasure[queryModel.getQueryMeasures().size()]));
+ blockExecutionInfo.setDataBlock(blockIndex);
+ blockExecutionInfo.setBlockKeyGenerator(blockKeyGenerator);
+ // adding aggregation info for query
+ blockExecutionInfo.setAggregatorInfo(getAggregatorInfoForBlock(queryModel, blockIndex));
+ // setting whether raw record query or not
+ blockExecutionInfo.setRawRecordDetailQuery(queryModel.isForcedDetailRawQuery());
+ // setting the masked byte of the block which will be
+ // used to update the unpack the older block keys
+ blockExecutionInfo.setMaskedByteForBlock(maksedByte);
+ // total number dimension
+ blockExecutionInfo
+ .setTotalNumberDimensionBlock(segmentProperties.getDimensionOrdinalToBlockMapping().size());
+ blockExecutionInfo
+ .setTotalNumberOfMeasureBlock(segmentProperties.getMeasuresOrdinalToBlockMapping().size());
+ blockExecutionInfo.setAbsoluteTableIdentifier(queryModel.getAbsoluteTableIdentifier());
+ blockExecutionInfo.setComplexDimensionInfoMap(QueryUtil
+ .getComplexDimensionsMap(updatedQueryDimension,
+ segmentProperties.getDimensionOrdinalToBlockMapping(),
+ segmentProperties.getEachComplexDimColumnValueSize(),
+ queryProperties.columnToDictionayMapping, queryProperties.complexFilterDimension));
+ // to check whether older block key update is required or not
+ blockExecutionInfo.setFixedKeyUpdateRequired(
+ !blockKeyGenerator.equals(queryProperties.keyStructureInfo.getKeyGenerator()));
+ IndexKey startIndexKey = null;
+ IndexKey endIndexKey = null;
+ if (null != queryModel.getFilterExpressionResolverTree()) {
+ // loading the filter executer tree for filter evaluation
+ blockExecutionInfo.setFilterExecuterTree(FilterUtil
+ .getFilterExecuterTree(queryModel.getFilterExpressionResolverTree(), segmentProperties,
+ blockExecutionInfo.getComlexDimensionInfoMap()));
+ List<IndexKey> listOfStartEndKeys = new ArrayList<IndexKey>(2);
+ FilterUtil.traverseResolverTreeAndGetStartAndEndKey(segmentProperties,
+ queryModel.getFilterExpressionResolverTree(), listOfStartEndKeys);
+ startIndexKey = listOfStartEndKeys.get(0);
+ endIndexKey = listOfStartEndKeys.get(1);
+ } else {
+ try {
+ startIndexKey = FilterUtil.prepareDefaultStartIndexKey(segmentProperties);
+ endIndexKey = FilterUtil.prepareDefaultEndIndexKey(segmentProperties);
+ } catch (KeyGenException e) {
+ throw new QueryExecutionException(e);
+ }
+ }
+ //setting the start index key of the block node
+ blockExecutionInfo.setStartKey(startIndexKey);
+ //setting the end index key of the block node
+ blockExecutionInfo.setEndKey(endIndexKey);
+ // expression dimensions
+ List<CarbonDimension> expressionDimensions =
+ new ArrayList<CarbonDimension>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ // expression measure
+ List<CarbonMeasure> expressionMeasures =
+ new ArrayList<CarbonMeasure>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ // setting all the dimension chunk indexes to be read from file
+ int numberOfElementToConsider = 0;
+ // list of dimensions to be projected
+ List<Integer> allProjectionListDimensionIdexes = new ArrayList<>();
+ int[] dimensionsBlockIndexes = QueryUtil.getDimensionsBlockIndexes(updatedQueryDimension,
+ segmentProperties.getDimensionOrdinalToBlockMapping(), expressionDimensions,
+ queryProperties.complexFilterDimension, allProjectionListDimensionIdexes);
+ if (dimensionsBlockIndexes.length > 0) {
+ numberOfElementToConsider = dimensionsBlockIndexes[dimensionsBlockIndexes.length - 1]
+ == segmentProperties.getBlockTodimensionOrdinalMapping().size() - 1 ?
+ dimensionsBlockIndexes.length - 1 :
+ dimensionsBlockIndexes.length;
+ blockExecutionInfo.setAllSelectedDimensionBlocksIndexes(CarbonUtil
+ .getRangeIndex(dimensionsBlockIndexes, numberOfElementToConsider,
+ CarbonCommonConstants.NUMBER_OF_COLUMN_READ_IN_IO));
+ } else {
+ blockExecutionInfo.setAllSelectedDimensionBlocksIndexes(new int[0][0]);
+ }
+
+ int[] measureBlockIndexes = QueryUtil
+ .getMeasureBlockIndexes(queryModel.getQueryMeasures(), expressionMeasures,
+ segmentProperties.getMeasuresOrdinalToBlockMapping(), queryProperties.filterMeasures);
+ if (measureBlockIndexes.length > 0) {
+
+ numberOfElementToConsider = measureBlockIndexes[measureBlockIndexes.length - 1]
+ == segmentProperties.getMeasures().size() - 1 ?
+ measureBlockIndexes.length - 1 :
+ measureBlockIndexes.length;
+ // setting all the measure chunk indexes to be read from file
+ blockExecutionInfo.setAllSelectedMeasureBlocksIndexes(CarbonUtil
+ .getRangeIndex(measureBlockIndexes, numberOfElementToConsider,
+ CarbonCommonConstants.NUMBER_OF_COLUMN_READ_IN_IO));
+ } else {
+ blockExecutionInfo.setAllSelectedMeasureBlocksIndexes(new int[0][0]);
+ }
+ // setting the key structure info which will be required
+ // to update the older block key with new key generator
+ blockExecutionInfo.setKeyStructureInfo(queryProperties.keyStructureInfo);
+ // setting the size of fixed key column (dictionary column)
+ blockExecutionInfo.setFixedLengthKeySize(getKeySize(updatedQueryDimension, segmentProperties));
+ Set<Integer> dictionaryColumnBlockIndex = new HashSet<Integer>();
+ List<Integer> noDictionaryColumnBlockIndex = new ArrayList<Integer>();
+ // get the block index to be read from file for query dimension
+ // for both dictionary columns and no dictionary columns
+ QueryUtil.fillQueryDimensionsBlockIndexes(updatedQueryDimension,
+ segmentProperties.getDimensionOrdinalToBlockMapping(), dictionaryColumnBlockIndex,
+ noDictionaryColumnBlockIndex);
+ int[] queryDictionaryColumnBlockIndexes = ArrayUtils.toPrimitive(
+ dictionaryColumnBlockIndex.toArray(new Integer[dictionaryColumnBlockIndex.size()]));
+ // need to sort the dictionary column as for all dimension
+ // column key will be filled based on key order
+ Arrays.sort(queryDictionaryColumnBlockIndexes);
+ blockExecutionInfo.setDictionaryColumnBlockIndex(queryDictionaryColumnBlockIndexes);
+ // setting the no dictionary column block indexes
+ blockExecutionInfo.setNoDictionaryBlockIndexes(ArrayUtils.toPrimitive(
+ noDictionaryColumnBlockIndex.toArray(new Integer[noDictionaryColumnBlockIndex.size()])));
+ // setting column id to dictionary mapping
+ blockExecutionInfo.setColumnIdToDcitionaryMapping(queryProperties.columnToDictionayMapping);
+ // setting each column value size
+ blockExecutionInfo.setEachColumnValueSize(segmentProperties.getEachDimColumnValueSize());
+ blockExecutionInfo.setComplexColumnParentBlockIndexes(
+ getComplexDimensionParentBlockIndexes(updatedQueryDimension));
+ blockExecutionInfo.setVectorBatchCollector(queryModel.isVectorReader());
+ try {
+ // to set column group and its key structure info which will be used
+ // to
+ // for getting the column group column data in case of final row
+ // and in case of dimension aggregation
+ blockExecutionInfo.setColumnGroupToKeyStructureInfo(
+ QueryUtil.getColumnGroupKeyStructureInfo(updatedQueryDimension, segmentProperties));
+ } catch (KeyGenException e) {
+ throw new QueryExecutionException(e);
+ }
+ return blockExecutionInfo;
+ }
+
+ /**
+ * This method will be used to get fixed key length size this will be used
+ * to create a row from column chunk
+ *
+ * @param queryDimension query dimension
+ * @param blockMetadataInfo block metadata info
+ * @return key size
+ */
+ private int getKeySize(List<QueryDimension> queryDimension, SegmentProperties blockMetadataInfo) {
+ List<Integer> fixedLengthDimensionOrdinal =
+ new ArrayList<Integer>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ int counter = 0;
+ while (counter < queryDimension.size()) {
+ if (queryDimension.get(counter).getDimension().numberOfChild() > 0) {
+ counter += queryDimension.get(counter).getDimension().numberOfChild();
+ continue;
+ } else if (!CarbonUtil.hasEncoding(queryDimension.get(counter).getDimension().getEncoder(),
+ Encoding.DICTIONARY)) {
+ counter++;
+ } else {
+ fixedLengthDimensionOrdinal.add(queryDimension.get(counter).getDimension().getKeyOrdinal());
+ counter++;
+ }
+ }
+ int[] dictioanryColumnOrdinal = ArrayUtils.toPrimitive(
+ fixedLengthDimensionOrdinal.toArray(new Integer[fixedLengthDimensionOrdinal.size()]));
+ if (dictioanryColumnOrdinal.length > 0) {
+ return blockMetadataInfo.getFixedLengthKeySplitter()
+ .getKeySizeByBlock(dictioanryColumnOrdinal);
+ }
+ return 0;
+ }
+
+ /**
+ * Below method will be used to get the aggrgator info for the query
+ *
+ * @param queryModel query model
+ * @param tableBlock table block
+ * @return aggregator info
+ */
+ private AggregatorInfo getAggregatorInfoForBlock(QueryModel queryModel,
+ AbstractIndex tableBlock) {
+ // getting the aggregate infos which will be used during aggregation
+ AggregatorInfo aggregatorInfos = RestructureUtil
+ .getAggregatorInfos(queryModel.getQueryMeasures(),
+ tableBlock.getSegmentProperties().getMeasures());
+ // setting the measure aggregator for all aggregation function selected
+ // in query
+ aggregatorInfos.setMeasureDataTypes(queryProperties.measureDataTypes);
+ return aggregatorInfos;
+ }
+
+ private int[] getComplexDimensionParentBlockIndexes(List<QueryDimension> queryDimensions) {
+ List<Integer> parentBlockIndexList = new ArrayList<Integer>();
+ for (QueryDimension queryDimension : queryDimensions) {
+ if (CarbonUtil.hasDataType(queryDimension.getDimension().getDataType(),
+ new DataType[] { DataType.ARRAY, DataType.STRUCT, DataType.MAP })) {
+ parentBlockIndexList.add(queryDimension.getDimension().getOrdinal());
+ }
+ }
+ return ArrayUtils
+ .toPrimitive(parentBlockIndexList.toArray(new Integer[parentBlockIndexList.size()]));
+ }
+
+ /**
+ * Below method will be used to finish the execution
+ *
+ * @throws QueryExecutionException
+ */
+ @Override public void finish() throws QueryExecutionException {
+ CarbonUtil.clearBlockCache(queryProperties.dataBlocks);
++ if(null != queryIterator) {
++ queryIterator.close();
++ }
+ if (null != queryProperties.executorService) {
+ queryProperties.executorService.shutdown();
+ try {
+ queryProperties.executorService.awaitTermination(1, TimeUnit.HOURS);
+ } catch (InterruptedException e) {
+ throw new QueryExecutionException(e);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f75ddfb6/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/DetailQueryExecutor.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/carbondata/core/scan/executor/impl/DetailQueryExecutor.java
index 46d0586,0000000..9c5b6a2
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/DetailQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/DetailQueryExecutor.java
@@@ -1,49 -1,0 +1,50 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.executor.impl;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.core.scan.result.BatchResult;
+import org.apache.carbondata.core.scan.result.iterator.DetailQueryResultIterator;
+
+/**
+ * Below class will be used to execute the detail query
+ * For executing the detail query it will pass all the block execution
+ * info to detail query result iterator and iterator will be returned
+ */
+public class DetailQueryExecutor extends AbstractQueryExecutor<BatchResult> {
+
+ @Override
+ public CarbonIterator<BatchResult> execute(QueryModel queryModel)
+ throws QueryExecutionException, IOException {
+ List<BlockExecutionInfo> blockExecutionInfoList = getBlockExecutionInfos(queryModel);
- return new DetailQueryResultIterator(
++ this.queryIterator = new DetailQueryResultIterator(
+ blockExecutionInfoList,
+ queryModel,
+ queryProperties.executorService
+ );
++ return queryIterator;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f75ddfb6/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/VectorDetailQueryExecutor.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/carbondata/core/scan/executor/impl/VectorDetailQueryExecutor.java
index 21dc6dd,0000000..cfea321
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/VectorDetailQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/VectorDetailQueryExecutor.java
@@@ -1,46 -1,0 +1,47 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.executor.impl;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.core.scan.result.iterator.VectorDetailQueryResultIterator;
+
+/**
+ * Below class will be used to execute the detail query and returns columnar vectors.
+ */
+public class VectorDetailQueryExecutor extends AbstractQueryExecutor<Object> {
+
+ @Override
+ public CarbonIterator<Object> execute(QueryModel queryModel)
+ throws QueryExecutionException, IOException {
+ List<BlockExecutionInfo> blockExecutionInfoList = getBlockExecutionInfos(queryModel);
- return new VectorDetailQueryResultIterator(
++ this.queryIterator = new VectorDetailQueryResultIterator(
+ blockExecutionInfoList,
+ queryModel,
+ queryProperties.executorService
+ );
++ return this.queryIterator;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f75ddfb6/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
index af73b69,0000000..9ba9797
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
@@@ -1,146 -1,0 +1,144 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.processor;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.scan.collector.ScannedResultCollector;
+import org.apache.carbondata.core.scan.collector.impl.DictionaryBasedResultCollector;
+import org.apache.carbondata.core.scan.collector.impl.DictionaryBasedVectorResultCollector;
+import org.apache.carbondata.core.scan.collector.impl.RawBasedResultCollector;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
+import org.apache.carbondata.core.scan.scanner.BlockletScanner;
+import org.apache.carbondata.core.scan.scanner.impl.FilterScanner;
+import org.apache.carbondata.core.scan.scanner.impl.NonFilterScanner;
+import org.apache.carbondata.core.stats.QueryStatisticsModel;
+
+/**
+ * This abstract class provides a skeletal implementation of the
+ * Block iterator.
+ */
+public abstract class AbstractDataBlockIterator extends CarbonIterator<List<Object[]>> {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(AbstractDataBlockIterator.class.getName());
+
+ /**
+ * iterator which will be used to iterate over data blocks
+ */
+ protected CarbonIterator<DataRefNode> dataBlockIterator;
+
+ /**
+ * result collector which will be used to aggregate the scanned result
+ */
+ protected ScannedResultCollector scannerResultAggregator;
+
+ /**
+ * processor which will be used to process the block processing can be
+ * filter processing or non filter processing
+ */
+ protected BlockletScanner blockletScanner;
+
+ /**
+ * to hold the data block
+ */
+ protected BlocksChunkHolder blocksChunkHolder;
+
+ /**
+ * batch size of result
+ */
+ protected int batchSize;
+
+ protected AbstractScannedResult scannedResult;
+
- public AbstractDataBlockIterator(BlockExecutionInfo blockExecutionInfo,
- FileHolder fileReader, int batchSize, QueryStatisticsModel queryStatisticsModel) {
++ public AbstractDataBlockIterator(BlockExecutionInfo blockExecutionInfo, FileHolder fileReader,
++ int batchSize, QueryStatisticsModel queryStatisticsModel,
++ BlocksChunkHolder blockChunkHolder) {
+ dataBlockIterator = new BlockletIterator(blockExecutionInfo.getFirstDataBlock(),
+ blockExecutionInfo.getNumberOfBlockToScan());
- blocksChunkHolder = new BlocksChunkHolder(blockExecutionInfo.getTotalNumberDimensionBlock(),
- blockExecutionInfo.getTotalNumberOfMeasureBlock());
- blocksChunkHolder.setFileReader(fileReader);
-
++ blocksChunkHolder = blockChunkHolder;
+ if (blockExecutionInfo.getFilterExecuterTree() != null) {
+ blockletScanner = new FilterScanner(blockExecutionInfo, queryStatisticsModel);
+ } else {
+ blockletScanner = new NonFilterScanner(blockExecutionInfo, queryStatisticsModel);
+ }
+ if (blockExecutionInfo.isRawRecordDetailQuery()) {
+ LOGGER.info("Row based raw collector is used to scan and collect the data");
+ this.scannerResultAggregator =
+ new RawBasedResultCollector(blockExecutionInfo);
+ } else if (blockExecutionInfo.isVectorBatchCollector()) {
+ LOGGER.info("Vector based dictionary collector is used to scan and collect the data");
+ this.scannerResultAggregator =
+ new DictionaryBasedVectorResultCollector(blockExecutionInfo);
+ } else {
+ LOGGER.info("Row based dictionary collector is used to scan and collect the data");
+ this.scannerResultAggregator =
+ new DictionaryBasedResultCollector(blockExecutionInfo);
+ }
+ this.batchSize = batchSize;
+ }
+
+ public boolean hasNext() {
+ if (scannedResult != null && scannedResult.hasNext()) {
+ return true;
+ } else {
+ return dataBlockIterator.hasNext();
+ }
+ }
+
+ protected boolean updateScanner() {
+ try {
+ if (scannedResult != null && scannedResult.hasNext()) {
+ return true;
+ } else {
+ scannedResult = getNextScannedResult();
+ while (scannedResult != null) {
+ if (scannedResult.hasNext()) {
+ return true;
+ }
+ scannedResult = getNextScannedResult();
+ }
+ return false;
+ }
+ } catch (IOException | FilterUnsupportedException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ private AbstractScannedResult getNextScannedResult()
+ throws IOException, FilterUnsupportedException {
+ if (dataBlockIterator.hasNext()) {
+ blocksChunkHolder.setDataBlock(dataBlockIterator.next());
+ blocksChunkHolder.reset();
+ return blockletScanner.scanBlocklet(blocksChunkHolder);
+ }
+ return null;
+ }
+
+ public abstract void processNextBatch(CarbonColumnarBatch columnarBatch);
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f75ddfb6/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java
index 48e3a0e,0000000..d5e5850
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java
@@@ -1,73 -1,0 +1,75 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.processor.impl;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.processor.AbstractDataBlockIterator;
++import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
+import org.apache.carbondata.core.stats.QueryStatisticsModel;
+
+/**
+ * Below class will be used to process the block for detail query
+ */
+public class DataBlockIteratorImpl extends AbstractDataBlockIterator {
+ /**
+ * DataBlockIteratorImpl Constructor
+ *
+ * @param blockExecutionInfo execution information
+ */
+ public DataBlockIteratorImpl(BlockExecutionInfo blockExecutionInfo, FileHolder fileReader,
- int batchSize, QueryStatisticsModel queryStatisticsModel) {
- super(blockExecutionInfo, fileReader, batchSize, queryStatisticsModel);
++ int batchSize, QueryStatisticsModel queryStatisticsModel,
++ BlocksChunkHolder blockChunkHolder) {
++ super(blockExecutionInfo, fileReader, batchSize, queryStatisticsModel, blockChunkHolder);
+ }
+
+ /**
+ * It scans the block and returns the result with @batchSize
+ *
+ * @return Result of @batchSize
+ */
+ public List<Object[]> next() {
+ List<Object[]> collectedResult = null;
+ if (updateScanner()) {
+ collectedResult = this.scannerResultAggregator.collectData(scannedResult, batchSize);
+ while (collectedResult.size() < batchSize && updateScanner()) {
+ List<Object[]> data = this.scannerResultAggregator
+ .collectData(scannedResult, batchSize - collectedResult.size());
+ collectedResult.addAll(data);
+ }
+ } else {
+ collectedResult = new ArrayList<>();
+ }
+ return collectedResult;
+ }
+
+ public void processNextBatch(CarbonColumnarBatch columnarBatch) {
+ if (updateScanner()) {
+ this.scannerResultAggregator.collectVectorBatch(scannedResult, columnarBatch);
+ while (columnarBatch.getActualSize() < columnarBatch.getBatchSize() && updateScanner()) {
+ this.scannerResultAggregator.collectVectorBatch(scannedResult, columnarBatch);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f75ddfb6/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
index cbe9e5f,0000000..bf2c51c
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
@@@ -1,570 -1,0 +1,569 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.result;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.Map;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
+import org.apache.carbondata.core.path.CarbonTablePath;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.filter.GenericQueryType;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
+import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
+import org.apache.carbondata.core.update.data.BlockletLevelDeleteDeltaDataCache;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+/**
+ * Scanned result class which will store and provide the result on request
+ */
+public abstract class AbstractScannedResult {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(AbstractScannedResult.class.getName());
+ /**
+ * current row number
+ */
+ protected int currentRow = -1;
+ /**
+ * row mapping indexes
+ */
+ protected int[] rowMapping;
+ /**
+ * key size of the fixed length column
+ */
+ private int fixedLengthKeySize;
+ /**
+ * total number of rows
+ */
+ private int totalNumberOfRows;
+ /**
+ * to keep track of number of rows process
+ */
+ protected int rowCounter;
+ /**
+ * dimension column data chunk
+ */
+ protected DimensionColumnDataChunk[] dataChunks;
+ /**
+ * measure column data chunk
+ */
+ protected MeasureColumnDataChunk[] measureDataChunks;
+ /**
+ * dictionary column block index in file
+ */
+ protected int[] dictionaryColumnBlockIndexes;
+
+ /**
+ * no dictionary column block index in file
+ */
+ protected int[] noDictionaryColumnBlockIndexes;
+
+ /**
+ * column group to is key structure info
+ * which will be used to get the key from the complete
+ * column group key
+ * For example if only one dimension of the column group is selected
+ * then from complete column group key it will be used to mask the key and
+ * get the particular column key
+ */
+ protected Map<Integer, KeyStructureInfo> columnGroupKeyStructureInfo;
+
+ /**
+ *
+ */
+ private Map<Integer, GenericQueryType> complexParentIndexToQueryMap;
+
+ private int totalDimensionsSize;
+
+ /**
+ * blockedId which will be blockId + blocklet number in the block
+ */
+ private String blockletId;
+
+ private long rowId;
+
+ /**
+ * parent block indexes
+ */
+ private int[] complexParentBlockIndexes;
+
+ protected BlockletLevelDeleteDeltaDataCache blockletDeleteDeltaCache;
+
+ public AbstractScannedResult(BlockExecutionInfo blockExecutionInfo) {
+ this.fixedLengthKeySize = blockExecutionInfo.getFixedLengthKeySize();
+ this.noDictionaryColumnBlockIndexes = blockExecutionInfo.getNoDictionaryBlockIndexes();
+ this.dictionaryColumnBlockIndexes = blockExecutionInfo.getDictionaryColumnBlockIndex();
+ this.columnGroupKeyStructureInfo = blockExecutionInfo.getColumnGroupToKeyStructureInfo();
+ this.complexParentIndexToQueryMap = blockExecutionInfo.getComlexDimensionInfoMap();
+ this.complexParentBlockIndexes = blockExecutionInfo.getComplexColumnParentBlockIndexes();
+ this.totalDimensionsSize = blockExecutionInfo.getQueryDimensions().length;
+ }
+
+ /**
+ * Below method will be used to set the dimension chunks
+ * which will be used to create a row
+ *
+ * @param dataChunks dimension chunks used in query
+ */
+ public void setDimensionChunks(DimensionColumnDataChunk[] dataChunks) {
+ this.dataChunks = dataChunks;
+ }
+
+ /**
+ * Below method will be used to set the measure column chunks
+ *
+ * @param measureDataChunks measure data chunks
+ */
+ public void setMeasureChunks(MeasureColumnDataChunk[] measureDataChunks) {
+ this.measureDataChunks = measureDataChunks;
+ }
+
+ /**
+ * Below method will be used to get the chunk based in measure ordinal
+ *
+ * @param ordinal measure ordinal
+ * @return measure column chunk
+ */
+ public MeasureColumnDataChunk getMeasureChunk(int ordinal) {
+ return measureDataChunks[ordinal];
+ }
+
+ /**
+ * Below method will be used to get the key for all the dictionary dimensions
+ * which is present in the query
+ *
+ * @param rowId row id selected after scanning
+ * @return return the dictionary key
+ */
+ protected byte[] getDictionaryKeyArray(int rowId) {
+ byte[] completeKey = new byte[fixedLengthKeySize];
+ int offset = 0;
+ for (int i = 0; i < this.dictionaryColumnBlockIndexes.length; i++) {
+ offset += dataChunks[dictionaryColumnBlockIndexes[i]]
+ .fillChunkData(completeKey, offset, rowId,
+ columnGroupKeyStructureInfo.get(dictionaryColumnBlockIndexes[i]));
+ }
+ rowCounter++;
+ return completeKey;
+ }
+
+ /**
+ * Below method will be used to get the key for all the dictionary dimensions
+ * in integer array format which is present in the query
+ *
+ * @param rowId row id selected after scanning
+ * @return return the dictionary key
+ */
+ protected int[] getDictionaryKeyIntegerArray(int rowId) {
+ int[] completeKey = new int[totalDimensionsSize];
+ int column = 0;
+ for (int i = 0; i < this.dictionaryColumnBlockIndexes.length; i++) {
+ column = dataChunks[dictionaryColumnBlockIndexes[i]]
+ .fillConvertedChunkData(rowId, column, completeKey,
+ columnGroupKeyStructureInfo.get(dictionaryColumnBlockIndexes[i]));
+ }
+ rowCounter++;
+ return completeKey;
+ }
+
+ /**
+ * Fill the column data of dictionary to vector
+ */
+ public void fillColumnarDictionaryBatch(ColumnVectorInfo[] vectorInfo) {
+ int column = 0;
+ for (int i = 0; i < this.dictionaryColumnBlockIndexes.length; i++) {
+ column = dataChunks[dictionaryColumnBlockIndexes[i]]
+ .fillConvertedChunkData(vectorInfo, column,
+ columnGroupKeyStructureInfo.get(dictionaryColumnBlockIndexes[i]));
+ }
+ }
+
+ /**
+ * Fill the column data to vector
+ */
+ public void fillColumnarNoDictionaryBatch(ColumnVectorInfo[] vectorInfo) {
+ int column = 0;
+ for (int i = 0; i < this.noDictionaryColumnBlockIndexes.length; i++) {
+ column = dataChunks[noDictionaryColumnBlockIndexes[i]]
+ .fillConvertedChunkData(vectorInfo, column,
+ columnGroupKeyStructureInfo.get(noDictionaryColumnBlockIndexes[i]));
+ }
+ }
+
+ /**
+ * Fill the measure column data to vector
+ */
+ public void fillColumnarMeasureBatch(ColumnVectorInfo[] vectorInfo, int[] measuresOrdinal) {
+ for (int i = 0; i < measuresOrdinal.length; i++) {
+ vectorInfo[i].measureVectorFiller
+ .fillMeasureVector(measureDataChunks[measuresOrdinal[i]], vectorInfo[i]);
+ }
+ }
+
+ public void fillColumnarComplexBatch(ColumnVectorInfo[] vectorInfos) {
+ for (int i = 0; i < vectorInfos.length; i++) {
+ int offset = vectorInfos[i].offset;
+ int len = offset + vectorInfos[i].size;
+ int vectorOffset = vectorInfos[i].vectorOffset;
+ CarbonColumnVector vector = vectorInfos[i].vector;
+ for (int j = offset; j < len; j++) {
+ ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+ DataOutputStream dataOutput = new DataOutputStream(byteStream);
+ try {
+ vectorInfos[i].genericQueryType.parseBlocksAndReturnComplexColumnByteArray(dataChunks,
+ rowMapping == null ? j : rowMapping[j], dataOutput);
+ Object data = vectorInfos[i].genericQueryType
+ .getDataBasedOnDataTypeFromSurrogates(ByteBuffer.wrap(byteStream.toByteArray()));
+ vector.putObject(vectorOffset++, data);
+ } catch (IOException e) {
+ LOGGER.error(e);
+ } finally {
+ CarbonUtil.closeStreams(dataOutput);
+ CarbonUtil.closeStreams(byteStream);
+ }
+ }
+ }
+ }
+
+ /**
+ * Just increment the counter incase of query only on measures.
+ */
+ public void incrementCounter() {
+ rowCounter++;
+ currentRow++;
+ }
+
+ /**
+ * increment the counter.
+ */
+ public void setRowCounter(int rowCounter) {
+ this.rowCounter = rowCounter;
+ }
+
+ /**
+ * Below method will be used to get the dimension data based on dimension
+ * ordinal and index
+ *
+ * @param dimOrdinal dimension ordinal present in the query
+ * @param rowId row index
+ * @return dimension data based on row id
+ */
+ protected byte[] getDimensionData(int dimOrdinal, int rowId) {
+ return dataChunks[dimOrdinal].getChunkData(rowId);
+ }
+
+ /**
+ * Below method will be used to get the dimension key array
+ * for all the no dictionary dimension present in the query
+ *
+ * @param rowId row number
+ * @return no dictionary keys for all no dictionary dimension
+ */
+ protected byte[][] getNoDictionaryKeyArray(int rowId) {
+ byte[][] noDictionaryColumnsKeys = new byte[noDictionaryColumnBlockIndexes.length][];
+ int position = 0;
+ for (int i = 0; i < this.noDictionaryColumnBlockIndexes.length; i++) {
+ noDictionaryColumnsKeys[position++] =
+ dataChunks[noDictionaryColumnBlockIndexes[i]].getChunkData(rowId);
+ }
+ return noDictionaryColumnsKeys;
+ }
+
+ /**
+ * Below method will be used to get the dimension key array
+ * for all the no dictionary dimension present in the query
+ *
+ * @param rowId row number
+ * @return no dictionary keys for all no dictionary dimension
+ */
+ protected String[] getNoDictionaryKeyStringArray(int rowId) {
+ String[] noDictionaryColumnsKeys = new String[noDictionaryColumnBlockIndexes.length];
+ int position = 0;
+ for (int i = 0; i < this.noDictionaryColumnBlockIndexes.length; i++) {
+ noDictionaryColumnsKeys[position++] =
+ new String(dataChunks[noDictionaryColumnBlockIndexes[i]].getChunkData(rowId));
+ }
+ return noDictionaryColumnsKeys;
+ }
+
+ /**
+ * @return blockletId
+ */
+ public String getBlockletId() {
+ return blockletId;
+ }
+
+ /**
+ * @param blockletId
+ */
+ public void setBlockletId(String blockletId) {
+ this.blockletId = CarbonTablePath.getShortBlockId(blockletId);
+ }
+
+ /**
+ * @return blockletId
+ */
+ public long getRowId() {
+ return rowId;
+ }
+
+ /**
+ * @param blockletId
+ */
+ public void setRowId(long rowId) {
+ this.rowId = rowId;
+ }
+
+
+ /**
+ * Below method will be used to get the complex type keys array based
+ * on row id for all the complex type dimension selected in query
+ *
+ * @param rowId row number
+ * @return complex type key array for all the complex dimension selected in query
+ */
+ protected byte[][] getComplexTypeKeyArray(int rowId) {
+ byte[][] complexTypeData = new byte[complexParentBlockIndexes.length][];
+ for (int i = 0; i < complexTypeData.length; i++) {
+ GenericQueryType genericQueryType =
+ complexParentIndexToQueryMap.get(complexParentBlockIndexes[i]);
+ ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+ DataOutputStream dataOutput = new DataOutputStream(byteStream);
+ try {
+ genericQueryType.parseBlocksAndReturnComplexColumnByteArray(dataChunks, rowId, dataOutput);
+ complexTypeData[i] = byteStream.toByteArray();
+ } catch (IOException e) {
+ LOGGER.error(e);
+ } finally {
+ CarbonUtil.closeStreams(dataOutput);
+ CarbonUtil.closeStreams(byteStream);
+ }
+ }
+ return complexTypeData;
+ }
+
+ /**
+ * @return return the total number of row after scanning
+ */
+ public int numberOfOutputRows() {
+ return this.totalNumberOfRows;
+ }
+
+ /**
+ * to check whether any more row is present in the result
+ *
+ * @return
+ */
+ public boolean hasNext() {
+ if (rowCounter < this.totalNumberOfRows) {
+ return true;
+ }
- CarbonUtil.freeMemory(dataChunks, measureDataChunks);
+ return false;
+ }
+
+ /**
+ * As this class will be a flyweight object so
+ * for one block all the blocklet scanning will use same result object
+ * in that case we need to reset the counter to zero so
+ * for new result it will give the result from zero
+ */
+ public void reset() {
+ rowCounter = 0;
+ currentRow = -1;
+ }
+
+ /**
+ * @param totalNumberOfRows set total of number rows valid after scanning
+ */
+ public void setNumberOfRows(int totalNumberOfRows) {
+ this.totalNumberOfRows = totalNumberOfRows;
+ }
+
+ /**
+ * After applying filter it will return the bit set with the valid row indexes
+ * so below method will be used to set the row indexes
+ *
+ * @param indexes
+ */
+ public void setIndexes(int[] indexes) {
+ this.rowMapping = indexes;
+ }
+
+ /**
+ * Below method will be used to check whether measure value is null or not
+ *
+ * @param ordinal measure ordinal
+ * @param rowIndex row number to be checked
+ * @return whether it is null or not
+ */
+ protected boolean isNullMeasureValue(int ordinal, int rowIndex) {
+ return measureDataChunks[ordinal].getNullValueIndexHolder().getBitSet().get(rowIndex);
+ }
+
+ /**
+ * Below method will be used to get the measure value of
+ * long type
+ *
+ * @param ordinal measure ordinal
+ * @param rowIndex row number of the measure value
+ * @return measure value of long type
+ */
+ protected long getLongMeasureValue(int ordinal, int rowIndex) {
+ return measureDataChunks[ordinal].getMeasureDataHolder().getReadableLongValueByIndex(rowIndex);
+ }
+
+ /**
+ * Below method will be used to get the measure value of double type
+ *
+ * @param ordinal measure ordinal
+ * @param rowIndex row number
+ * @return measure value of double type
+ */
+ protected double getDoubleMeasureValue(int ordinal, int rowIndex) {
+ return measureDataChunks[ordinal].getMeasureDataHolder()
+ .getReadableDoubleValueByIndex(rowIndex);
+ }
+
+ /**
+ * Below method will be used to get the measure type of big decimal data type
+ *
+ * @param ordinal ordinal of the of the measure
+ * @param rowIndex row number
+ * @return measure of big decimal type
+ */
+ protected BigDecimal getBigDecimalMeasureValue(int ordinal, int rowIndex) {
+ return measureDataChunks[ordinal].getMeasureDataHolder()
+ .getReadableBigDecimalValueByIndex(rowIndex);
+ }
+
+ public int getRowCounter() {
+ return rowCounter;
+ }
+
+ /**
+ * will return the current valid row id
+ *
+ * @return valid row id
+ */
+ public abstract int getCurrenrRowId();
+
+ /**
+ * @return dictionary key array for all the dictionary dimension
+ * selected in query
+ */
+ public abstract byte[] getDictionaryKeyArray();
+
+ /**
+ * @return dictionary key array for all the dictionary dimension in integer array forat
+ * selected in query
+ */
+ public abstract int[] getDictionaryKeyIntegerArray();
+
+ /**
+ * Return the dimension data based on dimension ordinal
+ *
+ * @param dimensionOrdinal dimension ordinal
+ * @return dimension data
+ */
+ public abstract byte[] getDimensionKey(int dimensionOrdinal);
+
+ /**
+ * Below method will be used to get the complex type key array
+ *
+ * @return complex type key array
+ */
+ public abstract byte[][] getComplexTypeKeyArray();
+
+ /**
+ * Below method will be used to get the no dictionary key
+ * array for all the no dictionary dimension selected in query
+ *
+ * @return no dictionary key array for all the no dictionary dimension
+ */
+ public abstract byte[][] getNoDictionaryKeyArray();
+
+ /**
+ * Below method will be used to get the no dictionary key
+ * array in string array format for all the no dictionary dimension selected in query
+ *
+ * @return no dictionary key array for all the no dictionary dimension
+ */
+ public abstract String[] getNoDictionaryKeyStringArray();
+
+ /**
+ * Below method will be used to to check whether measure value
+ * is null or for a measure
+ *
+ * @param ordinal measure ordinal
+ * @return is null or not
+ */
+ public abstract boolean isNullMeasureValue(int ordinal);
+
+ /**
+ * Below method will be used to get the measure value for measure
+ * of long data type
+ *
+ * @param ordinal measure ordinal
+ * @return long value of measure
+ */
+ public abstract long getLongMeasureValue(int ordinal);
+
+ /**
+ * Below method will be used to get the value of measure of double
+ * type
+ *
+ * @param ordinal measure ordinal
+ * @return measure value
+ */
+ public abstract double getDoubleMeasureValue(int ordinal);
+
+ /**
+ * Below method will be used to get the data of big decimal type
+ * of a measure
+ *
+ * @param ordinal measure ordinal
+ * @return measure value
+ */
+ public abstract BigDecimal getBigDecimalMeasureValue(int ordinal);
+
+ /**
+ *
+ * @return BlockletLevelDeleteDeltaDataCache.
+ */
+ public BlockletLevelDeleteDeltaDataCache getDeleteDeltaDataCache() {
+ return blockletDeleteDeltaCache;
+ }
+
+ /**
+ * @param blockletDeleteDeltaCache
+ */
+ public void setBlockletDeleteDeltaCache(
+ BlockletLevelDeleteDeltaDataCache blockletDeleteDeltaCache) {
+ this.blockletDeleteDeltaCache = blockletDeleteDeltaCache;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f75ddfb6/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
index 0ae02e1,0000000..8255ddc
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
@@@ -1,185 -1,0 +1,199 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.result.iterator;
+
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+
+import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.datastore.DataRefNodeFinder;
+import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.datastore.impl.btree.BTreeDataRefNodeFinder;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.core.scan.processor.AbstractDataBlockIterator;
+import org.apache.carbondata.core.scan.processor.impl.DataBlockIteratorImpl;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
+import org.apache.carbondata.core.stats.QueryStatistic;
+import org.apache.carbondata.core.stats.QueryStatisticsConstants;
+import org.apache.carbondata.core.stats.QueryStatisticsModel;
+import org.apache.carbondata.core.stats.QueryStatisticsRecorder;
+import org.apache.carbondata.core.util.CarbonProperties;
++import org.apache.carbondata.core.util.CarbonUtil;
+
+/**
+ * In case of detail query we cannot keep all the records in memory so for
+ * executing that query are returning a iterator over block and every time next
+ * call will come it will execute the block and return the result
+ */
+public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterator<E> {
+
+ /**
+ * LOGGER.
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(AbstractDetailQueryResultIterator.class.getName());
+
+ protected ExecutorService execService;
+ /**
+ * execution info of the block
+ */
+ protected List<BlockExecutionInfo> blockExecutionInfos;
+
+ /**
+ * file reader which will be used to execute the query
+ */
+ protected FileHolder fileReader;
+ protected AbstractDataBlockIterator dataBlockIterator;
+ protected boolean nextBatch = false;
+ /**
+ * total time scan the blocks
+ */
+ protected long totalScanTime;
+ /**
+ * is the statistic recorded
+ */
+ protected boolean isStatisticsRecorded;
+ /**
+ * QueryStatisticsRecorder
+ */
+ protected QueryStatisticsRecorder recorder;
+ /**
+ * number of cores which can be used
+ */
+ private int batchSize;
+ /**
+ * queryStatisticsModel to store query statistics object
+ */
+ QueryStatisticsModel queryStatisticsModel;
+
++ private BlocksChunkHolder blocksChunkHolder;
++
+ public AbstractDetailQueryResultIterator(List<BlockExecutionInfo> infos, QueryModel queryModel,
+ ExecutorService execService) {
+ String batchSizeString =
+ CarbonProperties.getInstance().getProperty(CarbonCommonConstants.DETAIL_QUERY_BATCH_SIZE);
+ if (null != batchSizeString) {
+ try {
+ batchSize = Integer.parseInt(batchSizeString);
+ } catch (NumberFormatException ne) {
+ LOGGER.error("Invalid inmemory records size. Using default value");
+ batchSize = CarbonCommonConstants.DETAIL_QUERY_BATCH_SIZE_DEFAULT;
+ }
+ } else {
+ batchSize = CarbonCommonConstants.DETAIL_QUERY_BATCH_SIZE_DEFAULT;
+ }
++ this.blocksChunkHolder = new BlocksChunkHolder(infos.get(0).getTotalNumberDimensionBlock(),
++ infos.get(0).getTotalNumberOfMeasureBlock());
+ this.recorder = queryModel.getStatisticsRecorder();
+ this.blockExecutionInfos = infos;
+ this.fileReader = FileFactory.getFileHolder(
+ FileFactory.getFileType(queryModel.getAbsoluteTableIdentifier().getStorePath()));
++ this.blocksChunkHolder.setFileReader(fileReader);
+ this.execService = execService;
+ intialiseInfos();
+ initQueryStatiticsModel();
+ }
+
+ private void intialiseInfos() {
+ totalScanTime = System.currentTimeMillis();
+ for (BlockExecutionInfo blockInfo : blockExecutionInfos) {
+ DataRefNodeFinder finder = new BTreeDataRefNodeFinder(blockInfo.getEachColumnValueSize());
+ DataRefNode startDataBlock = finder
+ .findFirstDataBlock(blockInfo.getDataBlock().getDataRefNode(), blockInfo.getStartKey());
+ while (startDataBlock.nodeNumber() < blockInfo.getStartBlockletIndex()) {
+ startDataBlock = startDataBlock.getNextDataRefNode();
+ }
+
+ long numberOfBlockToScan = blockInfo.getNumberOfBlockletToScan();
+ //if number of block is less than 0 then take end block.
+ if (numberOfBlockToScan <= 0) {
+ DataRefNode endDataBlock = finder
+ .findLastDataBlock(blockInfo.getDataBlock().getDataRefNode(), blockInfo.getEndKey());
+ numberOfBlockToScan = endDataBlock.nodeNumber() - startDataBlock.nodeNumber() + 1;
+ }
+ blockInfo.setFirstDataBlock(startDataBlock);
+ blockInfo.setNumberOfBlockToScan(numberOfBlockToScan);
+ }
+ }
+
+ @Override public boolean hasNext() {
+ if ((dataBlockIterator != null && dataBlockIterator.hasNext()) || nextBatch) {
+ return true;
+ } else if (blockExecutionInfos.size() > 0) {
+ return true;
+ } else {
+ if (!isStatisticsRecorded) {
+ QueryStatistic statistic = new QueryStatistic();
+ statistic.addFixedTimeStatistic(QueryStatisticsConstants.SCAN_BLOCKS_TIME,
+ System.currentTimeMillis() - totalScanTime);
+ recorder.recordStatistics(statistic);
+ isStatisticsRecorded = true;
+ }
+ return false;
+ }
+ }
+
+ protected void updateDataBlockIterator() {
+ if (dataBlockIterator == null || !dataBlockIterator.hasNext()) {
+ dataBlockIterator = getDataBlockIterator();
+ while (dataBlockIterator != null && !dataBlockIterator.hasNext()) {
+ dataBlockIterator = getDataBlockIterator();
+ }
+ }
+ }
+
+ private DataBlockIteratorImpl getDataBlockIterator() {
+ if (blockExecutionInfos.size() > 0) {
+ BlockExecutionInfo executionInfo = blockExecutionInfos.get(0);
+ blockExecutionInfos.remove(executionInfo);
+ queryStatisticsModel.setRecorder(recorder);
- return new DataBlockIteratorImpl(executionInfo, fileReader, batchSize, queryStatisticsModel);
++ CarbonUtil.freeMemory(blocksChunkHolder.getDimensionDataChunk(),
++ blocksChunkHolder.getMeasureDataChunk());
++ return new DataBlockIteratorImpl(executionInfo, fileReader, batchSize, queryStatisticsModel,
++ blocksChunkHolder);
+ }
+ return null;
+ }
+
+ protected void initQueryStatiticsModel() {
+ this.queryStatisticsModel = new QueryStatisticsModel();
+ QueryStatistic queryStatisticTotalBlocklet = new QueryStatistic();
+ queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .put(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM, queryStatisticTotalBlocklet);
+ QueryStatistic queryStatisticValidScanBlocklet = new QueryStatistic();
+ queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .put(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM, queryStatisticValidScanBlocklet);
+ }
+
+ public void processNextBatch(CarbonColumnarBatch columnarBatch) {
+ throw new UnsupportedOperationException("Please use VectorDetailQueryResultIterator");
+ }
+
++ @Override public void close() {
++ CarbonUtil.freeMemory(blocksChunkHolder.getDimensionDataChunk(),
++ blocksChunkHolder.getMeasureDataChunk());
++ }
++
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f75ddfb6/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
index 6787851,0000000..ac2683a
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
@@@ -1,200 -1,0 +1,202 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.scanner.impl;
+
+import java.io.IOException;
+import java.util.BitSet;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.result.impl.FilterQueryScannedResult;
+import org.apache.carbondata.core.scan.scanner.AbstractBlockletScanner;
+import org.apache.carbondata.core.stats.QueryStatistic;
+import org.apache.carbondata.core.stats.QueryStatisticsConstants;
+import org.apache.carbondata.core.stats.QueryStatisticsModel;
+import org.apache.carbondata.core.update.data.BlockletDeleteDeltaCacheLoader;
+import org.apache.carbondata.core.update.data.DeleteDeltaCacheLoaderIntf;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+/**
+ * Below class will be used for filter query processing
+ * this class will be first apply the filter then it will read the block if
+ * required and return the scanned result
+ */
+public class FilterScanner extends AbstractBlockletScanner {
+
+ /**
+ * filter tree
+ */
+ private FilterExecuter filterExecuter;
+ /**
+ * this will be used to apply min max
+ * this will be useful for dimension column which is on the right side
+ * as node finder will always give tentative blocks, if column data stored individually
+ * and data is in sorted order then we can check whether filter is in the range of min max or not
+ * if it present then only we can apply filter on complete data.
+ * this will be very useful in case of sparse data when rows are
+ * repeating.
+ */
+ private boolean isMinMaxEnabled;
+
+ private QueryStatisticsModel queryStatisticsModel;
+
+ public FilterScanner(BlockExecutionInfo blockExecutionInfo,
+ QueryStatisticsModel queryStatisticsModel) {
+ super(blockExecutionInfo);
+ scannedResult = new FilterQueryScannedResult(blockExecutionInfo);
+ // to check whether min max is enabled or not
+ String minMaxEnableValue = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.CARBON_QUERY_MIN_MAX_ENABLED,
+ CarbonCommonConstants.MIN_MAX_DEFAULT_VALUE);
+ if (null != minMaxEnableValue) {
+ isMinMaxEnabled = Boolean.parseBoolean(minMaxEnableValue);
+ }
+ // get the filter tree
+ this.filterExecuter = blockExecutionInfo.getFilterExecuterTree();
+ this.queryStatisticsModel = queryStatisticsModel;
+ }
+
+ /**
+ * Below method will be used to process the block
+ *
+ * @param blocksChunkHolder block chunk holder which holds the data
+ * @throws FilterUnsupportedException
+ */
+ @Override public AbstractScannedResult scanBlocklet(BlocksChunkHolder blocksChunkHolder)
+ throws IOException, FilterUnsupportedException {
+ fillScannedResult(blocksChunkHolder);
+ return scannedResult;
+ }
+
+ /**
+ * This method will process the data in below order
+ * 1. first apply min max on the filter tree and check whether any of the filter
+ * is fall on the range of min max, if not then return empty result
+ * 2. If filter falls on min max range then apply filter on actual
+ * data and get the filtered row index
+ * 3. if row index is empty then return the empty result
+ * 4. if row indexes is not empty then read only those blocks(measure or dimension)
+ * which was present in the query but not present in the filter, as while applying filter
+ * some of the blocks where already read and present in chunk holder so not need to
+ * read those blocks again, this is to avoid reading of same blocks which was already read
+ * 5. Set the blocks and filter indexes to result
+ *
+ * @param blocksChunkHolder
+ * @throws FilterUnsupportedException
+ */
+ private void fillScannedResult(BlocksChunkHolder blocksChunkHolder)
+ throws FilterUnsupportedException, IOException {
+ scannedResult.reset();
+ scannedResult.setBlockletId(
+ blockExecutionInfo.getBlockId() + CarbonCommonConstants.FILE_SEPARATOR + blocksChunkHolder
+ .getDataBlock().nodeNumber());
+ // apply min max
+ if (isMinMaxEnabled) {
+ BitSet bitSet = this.filterExecuter
+ .isScanRequired(blocksChunkHolder.getDataBlock().getColumnsMaxValue(),
+ blocksChunkHolder.getDataBlock().getColumnsMinValue());
+ if (bitSet.isEmpty()) {
+ scannedResult.setNumberOfRows(0);
+ scannedResult.setIndexes(new int[0]);
+ CarbonUtil.freeMemory(blocksChunkHolder.getDimensionDataChunk(),
+ blocksChunkHolder.getMeasureDataChunk());
+ return;
+ }
+ }
+ // apply filter on actual data
+ BitSet bitSet = this.filterExecuter.applyFilter(blocksChunkHolder);
+ // if indexes is empty then return with empty result
+ if (bitSet.isEmpty()) {
+ scannedResult.setNumberOfRows(0);
+ scannedResult.setIndexes(new int[0]);
++ CarbonUtil.freeMemory(blocksChunkHolder.getDimensionDataChunk(),
++ blocksChunkHolder.getMeasureDataChunk());
+ return;
+ }
+ // valid scanned blocklet
+ QueryStatistic validScannedBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM);
+ validScannedBlockletStatistic
+ .addCountStatistic(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM,
+ validScannedBlockletStatistic.getCount() + 1);
+ queryStatisticsModel.getRecorder().recordStatistics(validScannedBlockletStatistic);
+ // get the row indexes from bot set
+ int[] indexes = new int[bitSet.cardinality()];
+ int index = 0;
+ for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) {
+ indexes[index++] = i;
+ }
+ // loading delete data cache in blockexecutioninfo instance
+ DeleteDeltaCacheLoaderIntf deleteCacheLoader =
+ new BlockletDeleteDeltaCacheLoader(scannedResult.getBlockletId(),
+ blocksChunkHolder.getDataBlock(), blockExecutionInfo.getAbsoluteTableIdentifier());
+ deleteCacheLoader.loadDeleteDeltaFileDataToCache();
+ scannedResult
+ .setBlockletDeleteDeltaCache(blocksChunkHolder.getDataBlock().getDeleteDeltaDataCache());
+ FileHolder fileReader = blocksChunkHolder.getFileReader();
+ int[][] allSelectedDimensionBlocksIndexes =
+ blockExecutionInfo.getAllSelectedDimensionBlocksIndexes();
+ DimensionColumnDataChunk[] projectionListDimensionChunk = blocksChunkHolder.getDataBlock()
+ .getDimensionChunks(fileReader, allSelectedDimensionBlocksIndexes);
+
+ DimensionColumnDataChunk[] dimensionColumnDataChunk =
+ new DimensionColumnDataChunk[blockExecutionInfo.getTotalNumberDimensionBlock()];
+ // read dimension chunk blocks from file which is not present
+ for (int i = 0; i < dimensionColumnDataChunk.length; i++) {
+ if (null != blocksChunkHolder.getDimensionDataChunk()[i]) {
+ dimensionColumnDataChunk[i] = blocksChunkHolder.getDimensionDataChunk()[i];
+ }
+ }
+ for (int i = 0; i < allSelectedDimensionBlocksIndexes.length; i++) {
+ System.arraycopy(projectionListDimensionChunk, allSelectedDimensionBlocksIndexes[i][0],
+ dimensionColumnDataChunk, allSelectedDimensionBlocksIndexes[i][0],
+ allSelectedDimensionBlocksIndexes[i][1] + 1 - allSelectedDimensionBlocksIndexes[i][0]);
+ }
+ MeasureColumnDataChunk[] measureColumnDataChunk =
+ new MeasureColumnDataChunk[blockExecutionInfo.getTotalNumberOfMeasureBlock()];
+ int[][] allSelectedMeasureBlocksIndexes =
+ blockExecutionInfo.getAllSelectedMeasureBlocksIndexes();
+ MeasureColumnDataChunk[] projectionListMeasureChunk = blocksChunkHolder.getDataBlock()
+ .getMeasureChunks(fileReader, allSelectedMeasureBlocksIndexes);
+ // read the measure chunk blocks which is not present
+ for (int i = 0; i < measureColumnDataChunk.length; i++) {
+ if (null != blocksChunkHolder.getMeasureDataChunk()[i]) {
+ measureColumnDataChunk[i] = blocksChunkHolder.getMeasureDataChunk()[i];
+ }
+ }
+ for (int i = 0; i < allSelectedMeasureBlocksIndexes.length; i++) {
+ System.arraycopy(projectionListMeasureChunk, allSelectedMeasureBlocksIndexes[i][0],
+ measureColumnDataChunk, allSelectedMeasureBlocksIndexes[i][0],
+ allSelectedMeasureBlocksIndexes[i][1] + 1 - allSelectedMeasureBlocksIndexes[i][0]);
+ }
+ scannedResult.setDimensionChunks(dimensionColumnDataChunk);
+ scannedResult.setIndexes(indexes);
+ scannedResult.setMeasureChunks(measureColumnDataChunk);
+ scannedResult.setNumberOfRows(indexes.length);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f75ddfb6/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
[33/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnGroupModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnGroupModel.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnGroupModel.java
new file mode 100644
index 0000000..7402842
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnGroupModel.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.datastorage.columnar;
+
+public class ColumnGroupModel {
+
+ /**
+ * number of columns in columnar block
+ */
+ private int[] columnSplit;
+
+ /**
+ * total number of columns
+ */
+ private int noOfColumnsStore;
+
+ /**
+ * whether given index is columnar or not
+ * true: columnar
+ * false: row block
+ */
+ private boolean[] columnarStore;
+
+ /**
+ * column groups
+ * e.g
+ * {{0,1,2},3,4,{5,6}}
+ */
+ private int[][] columnGroups;
+
+ /**
+ * return columnSplit
+ *
+ * @return
+ */
+ public int[] getColumnSplit() {
+ return columnSplit;
+ }
+
+ /**
+ * set columnSplit
+ *
+ * @param split
+ */
+ public void setColumnSplit(int[] split) {
+ this.columnSplit = split;
+ }
+
+ /**
+ * @return no of columnar block
+ */
+ public int getNoOfColumnStore() {
+ return this.noOfColumnsStore;
+ }
+
+ /**
+ * set no of columnar block
+ *
+ * @param noOfColumnsStore
+ */
+ public void setNoOfColumnStore(int noOfColumnsStore) {
+ this.noOfColumnsStore = noOfColumnsStore;
+ }
+
+ /**
+ * it's an identifier for row block or single column block
+ *
+ * @param columnarStore
+ */
+ public void setColumnarStore(boolean[] columnarStore) {
+ this.columnarStore = columnarStore;
+ }
+
+ /**
+ * set column groups
+ *
+ * @param columnGroups
+ */
+ public void setColumnGroup(int[][] columnGroups) {
+ this.columnGroups = columnGroups;
+ }
+
+ /**
+ * check if given column group is columnar
+ *
+ * @param colGroup
+ * @return true if given block is columnar
+ */
+ public boolean isColumnar(int colGroup) {
+ return columnarStore[colGroup];
+ }
+
+ /**
+ * @return columngroups
+ */
+ public int[][] getColumnGroup() {
+ return this.columnGroups;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnWithIntIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnWithIntIndex.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnWithIntIndex.java
new file mode 100644
index 0000000..d569572
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnWithIntIndex.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.columnar;
+
+import java.util.Arrays;
+
+import org.apache.carbondata.core.util.ByteUtil;
+
+public class ColumnWithIntIndex implements Comparable<ColumnWithIntIndex> {
+ protected byte[] column;
+
+ private int index;
+
+ public ColumnWithIntIndex(byte[] column, int index) {
+ this.column = column;
+ this.index = index;
+ }
+
+ public ColumnWithIntIndex() {
+ }
+
+ /**
+ * @return the column
+ */
+ public byte[] getColumn() {
+ return column;
+ }
+
+ /**
+ * @param column the column to set
+ */
+ public void setColumn(byte[] column) {
+ this.column = column;
+ }
+
+ /**
+ * @return the index
+ */
+ public int getIndex() {
+ return index;
+ }
+
+ /**
+ * @param index the index to set
+ */
+ public void setIndex(int index) {
+ this.index = index;
+ }
+
+ @Override public int compareTo(ColumnWithIntIndex o) {
+ return ByteUtil.UnsafeComparer.INSTANCE.compareTo(column, o.column);
+ }
+
+ @Override public boolean equals(Object obj) {
+ if(obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ ColumnWithIntIndex o = (ColumnWithIntIndex)obj;
+ return Arrays.equals(column, o.column) && index == o.index;
+ }
+
+ @Override public int hashCode() {
+ return Arrays.hashCode(column) + index;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnWithIntIndexForHighCard.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnWithIntIndexForHighCard.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnWithIntIndexForHighCard.java
new file mode 100644
index 0000000..61145f7
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnWithIntIndexForHighCard.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.columnar;
+
+import java.util.Arrays;
+
+import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
+
+public class ColumnWithIntIndexForHighCard extends ColumnWithIntIndex
+ implements Comparable<ColumnWithIntIndex> {
+
+ public ColumnWithIntIndexForHighCard(byte[] column, int index) {
+ super(column, index);
+ }
+
+ @Override public int compareTo(ColumnWithIntIndex o) {
+ return UnsafeComparer.INSTANCE
+ .compareTo(column, 2, column.length - 2, o.column, 2, o.column.length - 2);
+ }
+
+ @Override public boolean equals(Object obj) {
+ if(obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ ColumnWithIntIndexForHighCard o = (ColumnWithIntIndexForHighCard)obj;
+ return Arrays.equals(column, o.column) && getIndex() == o.getIndex();
+ }
+
+ @Override public int hashCode() {
+ return Arrays.hashCode(column) + getIndex();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreDataHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreDataHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreDataHolder.java
new file mode 100644
index 0000000..31bfd00
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreDataHolder.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.columnar;
+
+import java.nio.ByteBuffer;
+
+public class ColumnarKeyStoreDataHolder {
+ private byte[] keyblockData;
+ private ColumnarKeyStoreMetadata columnarKeyStoreMetadata;
+
+ public ColumnarKeyStoreDataHolder(final byte[] keyblockData,
+ final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) {
+ this.keyblockData = keyblockData;
+ this.columnarKeyStoreMetadata = columnarKeyStoreMetadata;
+ }
+
+ public ColumnarKeyStoreDataHolder(final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) {
+ this.columnarKeyStoreMetadata = columnarKeyStoreMetadata;
+ }
+
+ public int getSurrogateKey(int columnIndex) {
+ byte[] actual = new byte[4];
+ int startIndex;
+ if (null != columnarKeyStoreMetadata.getColumnReverseIndex()) {
+ startIndex =
+ columnarKeyStoreMetadata.getColumnReverseIndex()[columnIndex] * columnarKeyStoreMetadata
+ .getEachRowSize();
+ } else {
+ startIndex = columnIndex * columnarKeyStoreMetadata.getEachRowSize();
+ }
+ int destPos = 4 - columnarKeyStoreMetadata.getEachRowSize();
+ System.arraycopy(keyblockData, startIndex, actual, destPos,
+ columnarKeyStoreMetadata.getEachRowSize());
+ return ByteBuffer.wrap(actual).getInt();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreMetadata.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreMetadata.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreMetadata.java
new file mode 100644
index 0000000..6badce5
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreMetadata.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.columnar;
+
+class ColumnarKeyStoreMetadata {
+
+ private int[] columnReverseIndex;
+
+ private int eachRowSize;
+
+ ColumnarKeyStoreMetadata(int eachRowSize) {
+ this.eachRowSize = eachRowSize;
+ }
+
+ /**
+ * @return the eachRowSize
+ */
+ int getEachRowSize() {
+ return eachRowSize;
+ }
+
+ /**
+ * @return the columnReverseIndex
+ */
+ int[] getColumnReverseIndex() {
+ return columnReverseIndex;
+ }
+
+ /**
+ * @param columnReverseIndex the columnReverseIndex to set
+ */
+ void setColumnReverseIndex(int[] columnReverseIndex) {
+ this.columnReverseIndex = columnReverseIndex;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/IndexStorage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/IndexStorage.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/IndexStorage.java
new file mode 100644
index 0000000..bc8b60e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/IndexStorage.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.columnar;
+
+public interface IndexStorage<T> {
+ boolean isAlreadySorted();
+
+ T getDataAfterComp();
+
+ T getIndexMap();
+
+ byte[][] getKeyBlock();
+
+ T getDataIndexMap();
+
+ int getTotalSize();
+
+ /**
+ * @return min value of block
+ */
+ byte[] getMin();
+
+ /**
+ * @return max value of block
+ */
+ byte[] getMax();
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/UnBlockIndexer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/UnBlockIndexer.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/UnBlockIndexer.java
new file mode 100644
index 0000000..4f2a80e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/UnBlockIndexer.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.columnar;
+
+import java.util.Arrays;
+
+public final class UnBlockIndexer {
+
+ private UnBlockIndexer() {
+
+ }
+
+ public static int[] uncompressIndex(int[] indexData, int[] indexMap) {
+ int actualSize = indexData.length;
+ for (int i = 0; i < indexMap.length; i++) {
+ actualSize += indexData[indexMap[i] + 1] - indexData[indexMap[i]] - 1;
+ }
+ int[] indexes = new int[actualSize];
+ int k = 0;
+ for (int i = 0; i < indexData.length; i++) {
+ int index = Arrays.binarySearch(indexMap, i);
+ if (index > -1) {
+ for (int j = indexData[indexMap[index]]; j <= indexData[indexMap[index] + 1]; j++) {
+ indexes[k] = j;
+ k++;
+ }
+ i++;
+ } else {
+ indexes[k] = indexData[i];
+ k++;
+ }
+ }
+ return indexes;
+ }
+
+ public static byte[] uncompressData(byte[] data, int[] index, int keyLen) {
+ if (index.length < 1) {
+ return data;
+ }
+ int numberOfCopy = 0;
+ int actualSize = 0;
+ int srcPos = 0;
+ int destPos = 0;
+ for (int i = 1; i < index.length; i += 2) {
+ actualSize += index[i];
+ }
+ byte[] uncompressedData = new byte[actualSize * keyLen];
+ int picIndex = 0;
+ for (int i = 0; i < data.length; i += keyLen) {
+ numberOfCopy = index[picIndex * 2 + 1];
+ picIndex++;
+ for (int j = 0; j < numberOfCopy; j++) {
+ System.arraycopy(data, srcPos, uncompressedData, destPos, keyLen);
+ destPos += keyLen;
+ }
+ srcPos += keyLen;
+ }
+ return uncompressedData;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/Compressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/Compressor.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/Compressor.java
new file mode 100644
index 0000000..7982d54
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/Compressor.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.compression;
+
+public interface Compressor {
+
+ byte[] compressByte(byte[] unCompInput);
+
+ byte[] unCompressByte(byte[] compInput);
+
+ byte[] unCompressByte(byte[] compInput, int offset, int length);
+
+ byte[] compressShort(short[] unCompInput);
+
+ short[] unCompressShort(byte[] compInput);
+
+ short[] unCompressShort(byte[] compInput, int offset, int lenght);
+
+ byte[] compressInt(int[] unCompInput);
+
+ int[] unCompressInt(byte[] compInput);
+
+ int[] unCompressInt(byte[] compInput, int offset, int length);
+
+ byte[] compressLong(long[] unCompInput);
+
+ long[] unCompressLong(byte[] compInput);
+
+ long[] unCompressLong(byte[] compInput, int offset, int length);
+
+ byte[] compressFloat(float[] unCompInput);
+
+ float[] unCompressFloat(byte[] compInput);
+
+ float[] unCompressFloat(byte[] compInput, int offset, int length);
+
+ byte[] compressDouble(double[] unCompInput);
+
+ double[] unCompressDouble(byte[] compInput);
+
+ double[] unCompressDouble(byte[] compInput, int offset, int length);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/CompressorFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/CompressorFactory.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/CompressorFactory.java
new file mode 100644
index 0000000..67c24d1
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/CompressorFactory.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.compression;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+public class CompressorFactory {
+
+ private static final CompressorFactory COMPRESSOR_FACTORY = new CompressorFactory();
+
+ private final Compressor compressor;
+
+ private CompressorFactory() {
+ String compressorType = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.COMPRESSOR, CarbonCommonConstants.DEFAULT_COMPRESSOR);
+ switch (compressorType) {
+ case "snappy":
+ compressor = new SnappyCompressor();
+ break;
+ default:
+ throw new RuntimeException(
+ "Invalid compressor type provided! Please provide valid compressor type");
+ }
+ }
+
+ public static CompressorFactory getInstance() {
+ return COMPRESSOR_FACTORY;
+ }
+
+ public Compressor getCompressor() {
+ return compressor;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/MeasureMetaDataModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/MeasureMetaDataModel.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/MeasureMetaDataModel.java
new file mode 100644
index 0000000..41a1ef0
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/MeasureMetaDataModel.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.compression;
+
+public class MeasureMetaDataModel {
+ /**
+ * maxValue
+ */
+ private Object[] maxValue;
+
+ /**
+ * minValue
+ */
+ private Object[] minValue;
+
+ /**
+ * mantissa
+ */
+ private int[] mantissa;
+
+ /**
+ * measureCount
+ */
+ private int measureCount;
+
+ /**
+ * uniqueValue
+ */
+ private Object[] uniqueValue;
+
+ /**
+ * type
+ */
+ private char[] type;
+
+ /**
+ * dataTypeSelected
+ */
+ private byte[] dataTypeSelected;
+
+ public MeasureMetaDataModel(Object[] minValue, Object[] maxValue, int[] mantissa,
+ int measureCount, Object[] uniqueValue, char[] type, byte[] dataTypeSelected) {
+ this.minValue = minValue;
+ this.maxValue = maxValue;
+ this.mantissa = mantissa;
+ this.measureCount = measureCount;
+ this.uniqueValue = uniqueValue;
+ this.type = type;
+ this.dataTypeSelected = dataTypeSelected;
+ }
+
+ /**
+ * get Max value
+ *
+ * @return
+ */
+ public Object[] getMaxValue() {
+ return maxValue;
+ }
+
+ /**
+ * getMinValue
+ *
+ * @return
+ */
+ public Object[] getMinValue() {
+ return minValue;
+ }
+
+ /**
+ * getMantissa
+ *
+ * @return
+ */
+ public int[] getMantissa() {
+ return mantissa;
+ }
+
+ /**
+ * getMeasureCount
+ *
+ * @return
+ */
+ public int getMeasureCount() {
+ return measureCount;
+ }
+
+ /**
+ * getUniqueValue
+ *
+ * @return
+ */
+ public Object[] getUniqueValue() {
+ return uniqueValue;
+ }
+
+ /**
+ * @return the type
+ */
+ public char[] getType() {
+ return type;
+ }
+
+ /**
+ * @return the dataTypeSelected
+ */
+ public byte[] getDataTypeSelected() {
+ return dataTypeSelected;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/ReaderCompressModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/ReaderCompressModel.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/ReaderCompressModel.java
new file mode 100644
index 0000000..02da265
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/ReaderCompressModel.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.compression;
+
+import org.apache.carbondata.core.metadata.ValueEncoderMeta;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+
+// Used in read path for decompression preparation
+public class ReaderCompressModel {
+ private ValueEncoderMeta valueEncoderMeta;
+
+ private ValueCompressionUtil.DataType convertedDataType;
+
+ private ValueCompressionHolder valueHolder;
+
+ public void setValueEncoderMeta(ValueEncoderMeta valueEncoderMeta) {
+ this.valueEncoderMeta = valueEncoderMeta;
+ }
+
+ public ValueCompressionUtil.DataType getConvertedDataType() {
+ return convertedDataType;
+ }
+
+ public void setConvertedDataType(ValueCompressionUtil.DataType convertedDataType) {
+ this.convertedDataType = convertedDataType;
+ }
+
+ public Object getMaxValue() {
+ return valueEncoderMeta.getMaxValue();
+ }
+
+ public int getMantissa() {
+ return valueEncoderMeta.getMantissa();
+ }
+
+ public ValueCompressionHolder getValueCompressionHolder() {
+ return valueHolder;
+ }
+
+ public void setValueCompressionHolder(ValueCompressionHolder valueHolder) {
+ this.valueHolder = valueHolder;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/SnappyCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/SnappyCompressor.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/SnappyCompressor.java
new file mode 100644
index 0000000..0ca6512
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/SnappyCompressor.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.compression;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+
+import org.xerial.snappy.Snappy;
+import org.xerial.snappy.SnappyNative;
+
+public class SnappyCompressor implements Compressor {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(SnappyCompressor.class.getName());
+
+ private final SnappyNative snappyNative;
+
+ public SnappyCompressor() {
+ Snappy snappy = new Snappy();
+ Field privateField = null;
+ try {
+ privateField = snappy.getClass().getDeclaredField("impl");
+ } catch (NoSuchFieldException | SecurityException e) {
+ throw new RuntimeException(e);
+ }
+ privateField.setAccessible(true);
+ try {
+ snappyNative = (SnappyNative) privateField.get(snappy);
+ } catch (IllegalArgumentException | IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override public byte[] compressByte(byte[] unCompInput) {
+ try {
+ return Snappy.rawCompress(unCompInput, unCompInput.length);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ return null;
+ }
+ }
+
+ @Override public byte[] unCompressByte(byte[] compInput) {
+ try {
+ return Snappy.uncompress(compInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return compInput;
+ }
+
+ @Override public byte[] unCompressByte(byte[] compInput, int offset, int length) {
+ int uncompressedLength = 0;
+ byte[] data = null;
+ try {
+ uncompressedLength = Snappy.uncompressedLength(compInput, offset, length);
+ data = new byte[uncompressedLength];
+ Snappy.uncompress(compInput, offset, length, data, 0);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return data;
+ }
+
+ @Override public byte[] compressShort(short[] unCompInput) {
+ try {
+ return Snappy.compress(unCompInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ return null;
+ }
+ }
+
+ @Override public short[] unCompressShort(byte[] compInput) {
+ try {
+ return Snappy.uncompressShortArray(compInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public short[] unCompressShort(byte[] compInput, int offset, int lenght) {
+ try {
+ return Snappy.uncompressShortArray(compInput, offset, lenght);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public byte[] compressInt(int[] unCompInput) {
+ try {
+ return Snappy.compress(unCompInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ return null;
+ }
+ }
+
+ @Override public int[] unCompressInt(byte[] compInput) {
+ try {
+ return Snappy.uncompressIntArray(compInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public int[] unCompressInt(byte[] compInput, int offset, int length) {
+ try {
+ return Snappy.uncompressIntArray(compInput, offset, length);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public byte[] compressLong(long[] unCompInput) {
+ try {
+ return Snappy.compress(unCompInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ return null;
+ }
+ }
+
+ @Override public long[] unCompressLong(byte[] compInput) {
+ try {
+ return Snappy.uncompressLongArray(compInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public long[] unCompressLong(byte[] compInput, int offset, int length) {
+ try {
+ return Snappy.uncompressLongArray(compInput, offset, length);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public byte[] compressFloat(float[] unCompInput) {
+ try {
+ return Snappy.compress(unCompInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ return null;
+ }
+ }
+
+ @Override public float[] unCompressFloat(byte[] compInput) {
+ try {
+ return Snappy.uncompressFloatArray(compInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public float[] unCompressFloat(byte[] compInput, int offset, int length) {
+ try {
+ return Snappy.uncompressFloatArray(compInput, offset, length);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public byte[] compressDouble(double[] unCompInput) {
+ try {
+ return Snappy.compress(unCompInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ return null;
+ }
+ }
+
+ @Override public double[] unCompressDouble(byte[] compInput) {
+ try {
+ return Snappy.uncompressDoubleArray(compInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public double[] unCompressDouble(byte[] compInput, int offset, int length) {
+ try {
+ int uncompressedLength = Snappy.uncompressedLength(compInput, offset, length);
+ double[] result = new double[uncompressedLength / 8];
+ snappyNative.rawUncompress(compInput, offset, length, result, 0);
+ return result;
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/ValueCompressionHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/ValueCompressionHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/ValueCompressionHolder.java
new file mode 100644
index 0000000..425c266
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/ValueCompressionHolder.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.compression;
+
+import java.math.BigDecimal;
+
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+
+
+/**
+ * ValueCompressionHolder is the base class for handling
+ * compression / decompression of the measure data chunk
+ */
+public abstract class ValueCompressionHolder<T> {
+
+ /**
+ * compressedValue
+ */
+ protected byte[] compressedValue;
+
+ /**
+ * @param compressor the compressor used to decompress the data
+ * @param dataType data type of the data
+ * @param data compressed data
+ */
+ public void unCompress(Compressor compressor, DataType dataType, byte[] data,
+ int offset, int length) {
+ switch (dataType) {
+ case DATA_BYTE:
+ setValue((T)compressor.unCompressByte(data, offset, length));
+ break;
+ case DATA_SHORT:
+ setValue((T)compressor.unCompressShort(data, offset, length));
+ break;
+ case DATA_INT:
+ setValue((T)compressor.unCompressInt(data, offset, length));
+ break;
+ case DATA_LONG:
+ case DATA_BIGINT:
+ setValue((T)compressor.unCompressLong(data, offset, length));
+ break;
+ case DATA_FLOAT:
+ setValue((T)compressor.unCompressFloat(data, offset, length));
+ break;
+ default:
+ setValue((T)compressor.unCompressDouble(data, offset, length));
+ break;
+ }
+ }
+
+ /**
+ * @param compressor the compressor used to compress the data
+ * @param dataType data type of the data
+ * @param data original data
+ */
+ public byte[] compress(Compressor compressor, DataType dataType, Object data) {
+ switch (dataType) {
+ case DATA_BYTE:
+ return compressor.compressByte((byte[])data);
+ case DATA_SHORT:
+ return compressor.compressShort((short[])data);
+ case DATA_INT:
+ return compressor.compressInt((int[])data);
+ case DATA_LONG:
+ case DATA_BIGINT:
+ return compressor.compressLong((long[])data);
+ case DATA_FLOAT:
+ return compressor.compressFloat((float[])data);
+ case DATA_DOUBLE:
+ default:
+ return compressor.compressDouble((double[])data);
+ }
+ }
+
+ public abstract void setValue(T value);
+
+ public abstract T getValue();
+
+ public abstract void setValueInBytes(byte[] value);
+
+ public abstract void compress();
+
+ public abstract void uncompress(DataType dataType, byte[] compressData, int offset,
+ int length, int decimal, Object maxValueObject);
+
+ public byte[] getCompressedData() { return compressedValue; }
+
+ public abstract long getLongValue(int index);
+
+ public abstract double getDoubleValue(int index);
+
+ public abstract BigDecimal getBigDecimalValue(int index);
+
+ public abstract void freeMemory();
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/WriterCompressModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/WriterCompressModel.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/WriterCompressModel.java
new file mode 100644
index 0000000..d833c61
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/WriterCompressModel.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.compression;
+
+import org.apache.carbondata.core.util.CompressionFinder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+
+public class WriterCompressModel {
+
+ /**
+ * DataType[] variable.
+ */
+ private ValueCompressionUtil.DataType[] convertedDataType;
+ /**
+ * DataType[] variable.
+ */
+ private ValueCompressionUtil.DataType[] actualDataType;
+
+ /**
+ * maxValue
+ */
+ private Object[] maxValue;
+ /**
+ * minValue.
+ */
+ private Object[] minValue;
+
+ /**
+ * uniqueValue
+ */
+ private Object[] uniqueValue;
+ /**
+ * mantissa.
+ */
+ private int[] mantissa;
+
+ /**
+ * aggType
+ */
+ private char[] type;
+
+ /**
+ * dataTypeSelected
+ */
+ private byte[] dataTypeSelected;
+ /**
+ * unCompressValues.
+ */
+ private ValueCompressionHolder[] valueHolder;
+
+ private CompressionFinder[] compressionFinders;
+
+ /**
+ * @return the convertedDataType
+ */
+ public ValueCompressionUtil.DataType[] getConvertedDataType() {
+ return convertedDataType;
+ }
+
+ /**
+ * @param convertedDataType the convertedDataType to set
+ */
+ public void setConvertedDataType(ValueCompressionUtil.DataType[] convertedDataType) {
+ this.convertedDataType = convertedDataType;
+ }
+
+ /**
+ * @return the actualDataType
+ */
+ public ValueCompressionUtil.DataType[] getActualDataType() {
+ return actualDataType;
+ }
+
+ /**
+ * @param actualDataType
+ */
+ public void setActualDataType(ValueCompressionUtil.DataType[] actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
+ /**
+ * @return the maxValue
+ */
+ public Object[] getMaxValue() {
+ return maxValue;
+ }
+
+ /**
+ * @param maxValue the maxValue to set
+ */
+ public void setMaxValue(Object[] maxValue) {
+ this.maxValue = maxValue;
+ }
+
+ /**
+ * @return the mantissa
+ */
+ public int[] getMantissa() {
+ return mantissa;
+ }
+
+ /**
+ * @param mantissa the mantissa to set
+ */
+ public void setMantissa(int[] mantissa) {
+ this.mantissa = mantissa;
+ }
+
+ /**
+ * getUnCompressValues().
+ *
+ * @return the unCompressValues
+ */
+ public ValueCompressionHolder[] getValueCompressionHolder() {
+ return valueHolder;
+ }
+
+ /**
+ * @param valueHolder set the ValueCompressionHolder
+ */
+ public void setValueCompressionHolder(ValueCompressionHolder[] valueHolder) {
+ this.valueHolder = valueHolder;
+ }
+
+ /**
+ * getMinValue
+ *
+ * @return
+ */
+ public Object[] getMinValue() {
+ return minValue;
+ }
+
+ /**
+ * setMinValue.
+ *
+ * @param minValue
+ */
+ public void setMinValue(Object[] minValue) {
+ this.minValue = minValue;
+ }
+
+ /**
+ * @return the aggType
+ */
+ public char[] getType() {
+ return type;
+ }
+
+ /**
+ * @param type the type to set
+ */
+ public void setType(char[] type) {
+ this.type = type;
+ }
+
+ /**
+ * @return the dataTypeSelected
+ */
+ public byte[] getDataTypeSelected() {
+ return dataTypeSelected;
+ }
+
+ /**
+ * @param dataTypeSelected the dataTypeSelected to set
+ */
+ public void setDataTypeSelected(byte[] dataTypeSelected) {
+ this.dataTypeSelected = dataTypeSelected;
+ }
+
+ /**
+ * getUniqueValue
+ *
+ * @return
+ */
+ public Object[] getUniqueValue() {
+ return uniqueValue;
+ }
+
+ /**
+ * setUniqueValue
+ *
+ * @param uniqueValue
+ */
+ public void setUniqueValue(Object[] uniqueValue) {
+ this.uniqueValue = uniqueValue;
+ }
+
+ public void setCompressionFinders(CompressionFinder[] compressionFinders) {
+ this.compressionFinders = compressionFinders;
+ }
+
+ public CompressionFinder[] getCompressionFinders() {
+ return this.compressionFinders;
+ }
+
+ /**
+ * @return the compType
+ */
+ public ValueCompressionUtil.COMPRESSION_TYPE getCompType(int index) {
+ return this.compressionFinders[index].getCompType();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinByte.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinByte.java
new file mode 100644
index 0000000..bd7ff59
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinByte.java
@@ -0,0 +1,115 @@
+/*
+3 * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.compression.decimal;
+
+import java.math.BigDecimal;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastorage.compression.Compressor;
+import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionMaxMinByte extends ValueCompressionHolder<byte[]> {
+
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionMaxMinByte.class.getName());
+
+ /**
+ * compressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+
+ /**
+ * value.
+ */
+ protected byte[] value;
+
+ private MeasureDataChunkStore<byte[]> measureChunkStore;
+
+ /**
+ * actual data type
+ */
+ protected DataType actualDataType;
+
+ private double maxValue;
+
+ public CompressionMaxMinByte(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
+ @Override public byte[] getValue() {return this.value; }
+
+ @Override public void setValue(byte[] value) {
+ this.value = value;
+ }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_BYTE, value);
+ }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, compressedData, offset, length);
+ setUncompressedValues(value, maxValueObject);
+
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ this.value = value;
+ }
+
+ @Override public long getLongValue(int index) {
+ byte byteValue = measureChunkStore.getByte(index);
+ return (long) (maxValue - byteValue);
+ }
+
+ @Override public double getDoubleValue(int index) {
+ byte byteValue = measureChunkStore.getByte(index);
+ return (maxValue - byteValue);
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal value is not defined for CompressionMaxMinByte");
+ }
+
+ private void setUncompressedValues(byte[] data, Object maxValueObject) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_BYTE, data.length);
+ this.measureChunkStore.putData(data);
+ if (maxValueObject instanceof Long) {
+ this.maxValue = (long)maxValueObject;
+ } else {
+ this.maxValue = (double) maxValueObject;
+ }
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinDefault.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinDefault.java
new file mode 100644
index 0000000..145f2ac
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinDefault.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.compression.decimal;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastorage.compression.Compressor;
+import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+
+public class CompressionMaxMinDefault extends ValueCompressionHolder<double[]> {
+
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionMaxMinDefault.class.getName());
+
+ /**
+ * compressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+
+ /**
+ * value.
+ */
+ private double[] value;
+
+ private MeasureDataChunkStore<double[]> measureChunkStore;
+
+ /**
+ * actual data type
+ */
+ private DataType actualDataType;
+
+ private double maxValue;
+
+ public CompressionMaxMinDefault(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
+ @Override public void setValue(double[] value) {
+ this.value = value;
+ }
+
+ @Override public double[] getValue() {return this.value; }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_DOUBLE, value);
+ }
+
+ @Override public void uncompress(DataType dataType, byte[] compressedData,
+ int offset, int length, int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, compressedData, offset, length);
+ setUncompressedValues(value, maxValueObject);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ ByteBuffer buffer = ByteBuffer.wrap(value);
+ this.value = ValueCompressionUtil.convertToDoubleArray(buffer, value.length);
+ }
+
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException(
+ "Long value is not defined for CompressionMaxMinDefault");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ double doubleValue = measureChunkStore.getDouble(index);
+ return maxValue - doubleValue;
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal value is not defined for CompressionMaxMinDefault");
+ }
+
+ private void setUncompressedValues(double[] data, Object maxValueObject) {
+ this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
+ .getMeasureDataChunkStore(DataType.DATA_DOUBLE, data.length);
+ this.measureChunkStore.putData(data);
+ if (maxValueObject instanceof Long) {
+ this.maxValue = (long) maxValueObject;
+ } else {
+ this.maxValue = (double) maxValueObject;
+ }
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinInt.java
new file mode 100644
index 0000000..9387155
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinInt.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.compression.decimal;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastorage.compression.Compressor;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionMaxMinInt extends ValueCompressionHolder<int[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionMaxMinInt.class.getName());
+
+ /**
+ * compressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+
+ private MeasureDataChunkStore<int[]> measureChunkStore;
+
+ /**
+ * value.
+ */
+ private int[] value;
+
+ private DataType actualDataType;
+
+ private double maxValue;
+
+ public CompressionMaxMinInt(DataType actualType) {
+ this.actualDataType = actualType;
+ }
+
+ @Override public void setValue(int[] value) {
+ this.value = value;
+ }
+
+ @Override public int[] getValue() { return this.value; }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, compressedData, offset, length);
+ setUncompressedValues(value, maxValueObject);
+ }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_INT, value);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ ByteBuffer buffer = ByteBuffer.wrap(value);
+ this.value = ValueCompressionUtil.convertToIntArray(buffer, value.length);
+ }
+
+ @Override public long getLongValue(int index) {
+ int intValue = measureChunkStore.getInt(index);
+ return (long) (maxValue - intValue);
+ }
+
+ @Override public double getDoubleValue(int index) {
+ int intValue = measureChunkStore.getInt(index);
+ return maxValue - intValue;
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal value is not defined for CompressionMaxMinInt");
+ }
+
+ private void setUncompressedValues(int[] data, Object maxValueObject) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, data.length);
+ this.measureChunkStore.putData(data);
+ if (maxValueObject instanceof Long) {
+ this.maxValue = (long) maxValueObject;
+ } else {
+ this.maxValue = (double) maxValueObject;
+ }
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinLong.java
new file mode 100644
index 0000000..d732e18
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinLong.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.compression.decimal;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastorage.compression.Compressor;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionMaxMinLong extends ValueCompressionHolder<long[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionMaxMinLong.class.getName());
+ /**
+ * compressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+
+ private MeasureDataChunkStore<long[]> measureChunkStore;
+
+ /**
+ * value.
+ */
+ protected long[] value;
+
+ protected DataType actualDataType;
+
+ private double maxValue;
+
+ public CompressionMaxMinLong(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_LONG, value);
+ }
+
+ @Override public void setValue(long[] value) {
+ this.value = value;
+
+ }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, compressedData, offset, length);
+ setUncompressValues(value, maxValueObject);
+ }
+
+ @Override public long[] getValue() {return this.value; }
+
+ @Override public void setValueInBytes(byte[] value) {
+ ByteBuffer buffer = ByteBuffer.wrap(value);
+ this.value = ValueCompressionUtil.convertToLongArray(buffer, value.length);
+ }
+
+ @Override public long getLongValue(int index) {
+ long longValue = measureChunkStore.getLong(index);
+ return (long) maxValue - longValue;
+ }
+
+ @Override public double getDoubleValue(int index) {
+ long longValue = measureChunkStore.getLong(index);
+ return maxValue - longValue;
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal value is not defined for CompressionMaxMinLong");
+ }
+
+ private void setUncompressValues(long[] data, Object maxValueObject) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_LONG, data.length);
+ this.measureChunkStore.putData(data);
+ if (maxValueObject instanceof Long) {
+ this.maxValue = (long) maxValueObject;
+ } else {
+ this.maxValue = (double) maxValueObject;
+ }
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinShort.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinShort.java
new file mode 100644
index 0000000..3f93978
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinShort.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.compression.decimal;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastorage.compression.Compressor;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionMaxMinShort extends ValueCompressionHolder<short[]> {
+
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionMaxMinShort.class.getName());
+
+ /**
+ * shortCompressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+
+ private MeasureDataChunkStore<short[]> measureChunkStore;
+
+ /**
+ * value.
+ */
+ private short[] value;
+
+ private DataType actualDataType;
+
+ private double maxValue;
+
+ public CompressionMaxMinShort(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
+ @Override public void setValue(short[] value) {
+ this.value = value;
+
+ }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, compressedData, offset, length);
+ setUncompressedValues(value, maxValueObject);
+ }
+
+ @Override public short[] getValue() {return this.value; }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_SHORT, value);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ ByteBuffer buffer = ByteBuffer.wrap(value);
+ this.value = ValueCompressionUtil.convertToShortArray(buffer, value.length);
+ }
+
+ @Override public long getLongValue(int index) {
+ short shortValue = measureChunkStore.getShort(index);
+ return (long) maxValue - shortValue;
+ }
+
+ @Override public double getDoubleValue(int index) {
+ short shortValue = measureChunkStore.getShort(index);
+ return maxValue - shortValue;
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal value is not defined for CompressionMaxMinShort");
+ }
+
+ private void setUncompressedValues(short[] data, Object maxValueObject) {
+ this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
+ .getMeasureDataChunkStore(DataType.DATA_SHORT, data.length);
+ this.measureChunkStore.putData(data);
+ if (maxValueObject instanceof Long) {
+ this.maxValue = (long) maxValueObject;
+ } else {
+ this.maxValue = (double) maxValueObject;
+ }
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalByte.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalByte.java
new file mode 100644
index 0000000..1767298
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalByte.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.compression.nondecimal;
+
+import java.math.BigDecimal;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastorage.compression.Compressor;
+import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNonDecimalByte extends ValueCompressionHolder<byte[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNonDecimalByte.class.getName());
+
+ /**
+ * compressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+
+ /**
+ * value.
+ */
+ private byte[] value;
+
+ private MeasureDataChunkStore<byte[]> measureChunkStore;
+
+ private double divisionFactory;
+
+ @Override public void setValue(byte[] value) {
+ this.value = value;
+ }
+
+ @Override public byte[] getValue() {return this.value; }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_BYTE, value);
+ }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] compressedData,
+ int offset, int length, int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, compressedData, offset, length);
+ setUncompressedValues(value, decimalPlaces);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ this.value = value;
+ }
+
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException(
+ "Long value is not defined for CompressionNonDecimalByte");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ return (measureChunkStore.getByte(index) / this.divisionFactory);
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal value is not defined for CompressionNonDecimalByte");
+ }
+
+ private void setUncompressedValues(byte[] data, int decimalPlaces) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_BYTE, data.length);
+ this.measureChunkStore.putData(data);
+ this.divisionFactory = Math.pow(10, decimalPlaces);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalDefault.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalDefault.java
new file mode 100644
index 0000000..ac0ec78
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalDefault.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.compression.nondecimal;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastorage.compression.Compressor;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNonDecimalDefault extends ValueCompressionHolder<double[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNonDecimalDefault.class.getName());
+ /**
+ * doubleCompressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+ /**
+ * value.
+ */
+ private double[] value;
+
+ private MeasureDataChunkStore<double[]> measureChunkStore;
+
+ private double divisionFactory;
+
+ @Override
+ public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, compressedData, offset, length);
+ setUncompressedValues(value, decimalPlaces);
+ }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_DOUBLE, value);
+ }
+
+ @Override public void setValue(double[] value) {
+ this.value = value;
+
+ }
+
+ @Override public double[] getValue() {return this.value; }
+
+ @Override public void setValueInBytes(byte[] value) {
+ ByteBuffer buffer = ByteBuffer.wrap(value);
+ this.value = ValueCompressionUtil.convertToDoubleArray(buffer, value.length);
+ }
+
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException(
+ "Long value is not defined for CompressionNonDecimalDefault");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ return (measureChunkStore.getDouble(index) / divisionFactory);
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal value is not defined for CompressionNonDecimalDefault");
+ }
+
+ private void setUncompressedValues(double[] data, int decimalPlaces) {
+ this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
+ .getMeasureDataChunkStore(DataType.DATA_DOUBLE, data.length);
+ this.measureChunkStore.putData(data);
+ this.divisionFactory = Math.pow(10, decimalPlaces);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalInt.java
new file mode 100644
index 0000000..1e5689d
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalInt.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.compression.nondecimal;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastorage.compression.Compressor;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNonDecimalInt extends ValueCompressionHolder<int[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNonDecimalInt.class.getName());
+ /**
+ * intCompressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+ /**
+ * value.
+ */
+ private int[] value;
+
+ private MeasureDataChunkStore<int[]> measureChunkStore;
+
+ private double divisionFactory;
+
+ @Override public void setValue(int[] value) {
+ this.value = value;
+ }
+
+ @Override public int[] getValue() { return this.value; }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_INT, value);
+ }
+
+ @Override public void setValueInBytes(byte[] bytesArr) {
+ ByteBuffer buffer = ByteBuffer.wrap(bytesArr);
+ this.value = ValueCompressionUtil.convertToIntArray(buffer, bytesArr.length);
+ }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, compressedData, offset, length);
+ setUncompressedValues(value, decimalPlaces);
+ }
+
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException(
+ "Long value is not defined for CompressionNonDecimalInt");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ return (measureChunkStore.getInt(index) / this.divisionFactory);
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal value is not defined for CompressionNonDecmialInt");
+ }
+
+ private void setUncompressedValues(int[] data, int decimalPlaces) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, data.length);
+ this.measureChunkStore.putData(data);
+ this.divisionFactory = Math.pow(10, decimalPlaces);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalLong.java
new file mode 100644
index 0000000..d84b8f2
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalLong.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.compression.nondecimal;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastorage.compression.Compressor;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNonDecimalLong extends ValueCompressionHolder<long[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNonDecimalLong.class.getName());
+
+ /**
+ * longCompressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+
+ /**
+ * value.
+ */
+ private long[] value;
+
+ private MeasureDataChunkStore<long[]> measureChunkStore;
+
+ private double divisionFactory;
+
+ @Override public void setValue(long[] value) {
+ this.value = value;
+ }
+
+ @Override public long[] getValue() { return this.value; }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_LONG, value);
+ }
+
+ @Override public void uncompress(DataType dataType, byte[] compressedData, int offset,
+ int length, int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, compressedData, offset, length);
+ setUncompressedValues(value, decimalPlaces);
+ }
+
+ @Override public void setValueInBytes(byte[] bytes) {
+ ByteBuffer buffer = ByteBuffer.wrap(bytes);
+ this.value = ValueCompressionUtil.convertToLongArray(buffer, bytes.length);
+ }
+
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException(
+ "Long value is not defined for CompressionNonDecimalLong");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ return (measureChunkStore.getLong(index) / this.divisionFactory);
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal value is not defined for CompressionNonDecimalLong");
+ }
+
+ private void setUncompressedValues(long[] data, int decimalPlaces) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_LONG, data.length);
+ this.measureChunkStore.putData(data);
+ this.divisionFactory = Math.pow(10, decimalPlaces);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
[22/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
new file mode 100644
index 0000000..6f2d3e2
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter.executer;
+
+import java.io.IOException;
+import java.util.BitSet;
+import java.util.List;
+
+import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
+import org.apache.carbondata.core.metadata.Encoding;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
+import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+
+public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecuterImpl {
+ private byte[][] filterRangeValues;
+
+ public RowLevelRangeLessThanFiterExecuterImpl(
+ List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
+ List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
+ AbsoluteTableIdentifier tableIdentifier, byte[][] filterRangeValues,
+ SegmentProperties segmentProperties) {
+ super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties,
+ null);
+ this.filterRangeValues = filterRangeValues;
+ }
+
+ @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ BitSet bitSet = new BitSet(1);
+ byte[][] filterValues = this.filterRangeValues;
+ int columnIndex = this.dimColEvaluatorInfoList.get(0).getColumnIndex();
+ boolean isScanRequired = false;
+ for (int k = 0; k < filterValues.length; k++) {
+ // and filter-min should be positive
+ int minCompare =
+ ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMinValue[columnIndex]);
+
+ // if any filter applied is not in range of min and max of block
+ // then since its a less than fiter validate whether the block
+ // min range is less than applied filter member
+ if (minCompare > 0) {
+ isScanRequired = true;
+ break;
+ }
+ }
+ if (isScanRequired) {
+ bitSet.set(0);
+ }
+ return bitSet;
+
+ }
+
+ @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
+ throws FilterUnsupportedException, IOException {
+ if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ return super.applyFilter(blockChunkHolder);
+ }
+ int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
+ .get(dimColEvaluatorInfoList.get(0).getColumnIndex());
+ if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
+ blockChunkHolder.getDimensionDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
+ return getFilteredIndexes(blockChunkHolder.getDimensionDataChunk()[blockIndex],
+ blockChunkHolder.getDataBlock().nodeSize());
+ }
+
+ private BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
+ int numerOfRows) {
+ byte[] defaultValue = null;
+ if (dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
+ .getDirectDictionaryGenerator(
+ dimColEvaluatorInfoList.get(0).getDimension().getDataType());
+ int key = directDictionaryGenerator.generateDirectSurrogateKey(null) + 1;
+ defaultValue = FilterUtil.getMaskKey(key, dimColEvaluatorInfoList.get(0).getDimension(),
+ this.segmentProperties.getDimensionKeyGenerator());
+ }
+ if (dimensionColumnDataChunk.isExplicitSorted()
+ && dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
+ return setFilterdIndexToBitSetWithColumnIndex(
+ (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, numerOfRows, defaultValue);
+ }
+ return setFilterdIndexToBitSet(dimensionColumnDataChunk, numerOfRows, defaultValue);
+ }
+
+ /**
+ * Method will scan the block and finds the range start index from which all members
+ * will be considered for applying range filters. this method will be called if the
+ * column is not supported by default so column index mapping will be present for
+ * accesing the members from the block.
+ *
+ * @param dimensionColumnDataChunk
+ * @param numerOfRows
+ * @return BitSet.
+ */
+ private BitSet setFilterdIndexToBitSetWithColumnIndex(
+ FixedLengthDimensionDataChunk dimensionColumnDataChunk, int numerOfRows,
+ byte[] defaultValue) {
+ BitSet bitSet = new BitSet(numerOfRows);
+ int start = 0;
+ int last = 0;
+ int startIndex = 0;
+ int skip = 0;
+ byte[][] filterValues = this.filterRangeValues;
+
+ //find the number of default values to skip the null value in case of direct dictionary
+ if (null != defaultValue) {
+ start = CarbonUtil
+ .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ defaultValue, false);
+ if (start < 0) {
+ skip = -(start + 1);
+ // end of block
+ if (skip == numerOfRows) {
+ return bitSet;
+ }
+ } else {
+ skip = start;
+ }
+ startIndex = skip;
+ }
+
+ for (int i = 0; i < filterValues.length; i++) {
+ start = CarbonUtil
+ .getFirstIndexUsingBinarySearch(dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ filterValues[i], false);
+ // Logic will handle the case where the range filter member is not present in block
+ // in this case the binary search will return the index from where the bit sets will be
+ // set inorder to apply filters. this is Lesser than filter so the range will be taken
+ // from the prev element which is Lesser than filter member.
+ start = CarbonUtil.nextLesserValueToTarget(start, dimensionColumnDataChunk, filterValues[i]);
+ if (start < 0) {
+ start = -(start + 1);
+ if (start == numerOfRows) {
+ start = start - 1;
+ }
+ // Method will compare the tentative index value after binary search, this tentative
+ // index needs to be compared by the filter member if its < filter then from that
+ // index the bitset will be considered for filtering process.
+ if (ByteUtil.compare(filterValues[i],
+ dimensionColumnDataChunk.getChunkData(dimensionColumnDataChunk.getInvertedIndex(start)))
+ < 0) {
+ start = start - 1;
+ }
+ }
+ last = start;
+ for (int j = start; j >= skip; j--) {
+ bitSet.set(dimensionColumnDataChunk.getInvertedIndex(j));
+ last--;
+ }
+ startIndex = last;
+ if (startIndex >= 0) {
+ break;
+ }
+ }
+ return bitSet;
+ }
+
+ /**
+ * Method will scan the block and finds the range start index from which all
+ * members will be considered for applying range filters. this method will
+ * be called if the column is sorted default so column index
+ * mapping will be present for accesing the members from the block.
+ *
+ * @param dimensionColumnDataChunk
+ * @param numerOfRows
+ * @return BitSet.
+ */
+ private BitSet setFilterdIndexToBitSet(DimensionColumnDataChunk dimensionColumnDataChunk,
+ int numerOfRows, byte[] defaultValue) {
+ BitSet bitSet = new BitSet(numerOfRows);
+ if (dimensionColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
+ int start = 0;
+ int last = 0;
+ int startIndex = 0;
+ int skip = 0;
+ byte[][] filterValues = this.filterRangeValues;
+ //find the number of default values to skip the null value in case of direct dictionary
+ if (null != defaultValue) {
+ start = CarbonUtil.getFirstIndexUsingBinarySearch(
+ (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ defaultValue, false);
+ if (start < 0) {
+ skip = -(start + 1);
+ // end of block
+ if (skip == numerOfRows) {
+ return bitSet;
+ }
+ } else {
+ skip = start;
+ }
+ startIndex = skip;
+ }
+ for (int k = 0; k < filterValues.length; k++) {
+ start = CarbonUtil.getFirstIndexUsingBinarySearch(
+ (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, startIndex, numerOfRows - 1,
+ filterValues[k], false);
+ if(start >= 0) {
+ start = CarbonUtil.nextLesserValueToTarget(start,
+ (FixedLengthDimensionDataChunk) dimensionColumnDataChunk, filterValues[k]);
+ }
+ if (start < 0) {
+ start = -(start + 1);
+
+ if (start >= numerOfRows) {
+ start = numerOfRows - 1;
+ }
+ // Method will compare the tentative index value after binary search, this tentative
+ // index needs to be compared by the filter member if its < filter then from that
+ // index the bitset will be considered for filtering process.
+ if (ByteUtil.compare(filterValues[k], dimensionColumnDataChunk.getChunkData(start)) < 0) {
+ start = start - 1;
+ }
+ }
+ last = start;
+ for (int j = start; j >= skip; j--) {
+ bitSet.set(j);
+ last--;
+ }
+ startIndex = last;
+ if (startIndex <= 0) {
+ break;
+ }
+ }
+ }
+ return bitSet;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java
new file mode 100644
index 0000000..d5e58c9
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeTypeExecuterFacory.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter.executer;
+
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+import org.apache.carbondata.core.scan.filter.resolver.RowLevelRangeFilterResolverImpl;
+
+public class RowLevelRangeTypeExecuterFacory {
+
+ private RowLevelRangeTypeExecuterFacory() {
+
+ }
+
+ /**
+ * The method returns the Row Level Range fiter type instance based on
+ * filter tree resolver type.
+ *
+ * @param filterExpressionResolverTree
+ * @param segmentProperties
+ * @return the generator instance
+ */
+ public static RowLevelFilterExecuterImpl getRowLevelRangeTypeExecuter(
+ FilterExecuterType filterExecuterType, FilterResolverIntf filterExpressionResolverTree,
+ SegmentProperties segmentProperties) {
+ switch (filterExecuterType) {
+
+ case ROWLEVEL_LESSTHAN:
+ return new RowLevelRangeLessThanFiterExecuterImpl(
+ ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
+ .getDimColEvaluatorInfoList(),
+ ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
+ .getMsrColEvalutorInfoList(),
+ filterExpressionResolverTree.getFilterExpression(),
+ ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(),
+ ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
+ .getFilterRangeValues(segmentProperties), segmentProperties);
+ case ROWLEVEL_LESSTHAN_EQUALTO:
+ return new RowLevelRangeLessThanEqualFilterExecuterImpl(
+ ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
+ .getDimColEvaluatorInfoList(),
+ ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
+ .getMsrColEvalutorInfoList(),
+ filterExpressionResolverTree.getFilterExpression(),
+ ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(),
+ ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
+ .getFilterRangeValues(segmentProperties), segmentProperties);
+ case ROWLEVEL_GREATERTHAN_EQUALTO:
+ return new RowLevelRangeGrtrThanEquaToFilterExecuterImpl(
+ ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
+ .getDimColEvaluatorInfoList(),
+ ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
+ .getMsrColEvalutorInfoList(),
+ filterExpressionResolverTree.getFilterExpression(),
+ ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(),
+ ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
+ .getFilterRangeValues(segmentProperties), segmentProperties);
+ case ROWLEVEL_GREATERTHAN:
+ return new RowLevelRangeGrtThanFiterExecuterImpl(
+ ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
+ .getDimColEvaluatorInfoList(),
+ ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
+ .getMsrColEvalutorInfoList(),
+ filterExpressionResolverTree.getFilterExpression(),
+ ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(),
+ ((RowLevelRangeFilterResolverImpl) filterExpressionResolverTree)
+ .getFilterRangeValues(segmentProperties), segmentProperties);
+ default:
+ // Scenario wont come logic must break
+ return null;
+
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/ExpressionType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/ExpressionType.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/ExpressionType.java
new file mode 100644
index 0000000..2ca481f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/ExpressionType.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.filter.intf;
+
+public enum ExpressionType {
+
+ AND,
+ OR,
+ NOT,
+ EQUALS,
+ NOT_EQUALS,
+ LESSTHAN,
+ LESSTHAN_EQUALTO,
+ GREATERTHAN,
+ GREATERTHAN_EQUALTO,
+ ADD,
+ SUBSTRACT,
+ DIVIDE,
+ MULTIPLY,
+ IN,
+ LIST,
+ NOT_IN,
+ UNKNOWN,
+ LITERAL,
+ RANGE,
+ FALSE
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterExecuterType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterExecuterType.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterExecuterType.java
new file mode 100644
index 0000000..eba0d8a
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterExecuterType.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter.intf;
+
+import java.io.Serializable;
+
+public enum FilterExecuterType implements Serializable {
+
+ INCLUDE, EXCLUDE, OR, AND, RESTRUCTURE, ROWLEVEL, RANGE, ROWLEVEL_GREATERTHAN,
+ ROWLEVEL_GREATERTHAN_EQUALTO, ROWLEVEL_LESSTHAN_EQUALTO, ROWLEVEL_LESSTHAN
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/RowImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/RowImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/RowImpl.java
new file mode 100644
index 0000000..a13b140
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/RowImpl.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.filter.intf;
+
+public class RowImpl implements RowIntf {
+ private Object[] row;
+
+ public RowImpl() {
+ row = new Object[0];
+ }
+
+ @Override public Object getVal(int index) {
+ return row[index];
+ }
+
+ @Override public Object[] getValues() {
+ return row;
+ }
+
+ @Override public void setValues(final Object[] row) {
+ this.row = row;
+ }
+
+ @Override public int size() {
+ return this.row.length;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/RowIntf.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/RowIntf.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/RowIntf.java
new file mode 100644
index 0000000..7640553
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/RowIntf.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.filter.intf;
+
+public interface RowIntf {
+ Object getVal(int index);
+
+ Object[] getValues();
+
+ void setValues(Object[] setValues);
+
+ int size();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
new file mode 100644
index 0000000..ce18b75
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter.resolver;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.SortedMap;
+
+import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.metadata.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.conditional.BinaryConditionalExpression;
+import org.apache.carbondata.core.scan.expression.conditional.ConditionalExpression;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType;
+import org.apache.carbondata.core.scan.filter.resolver.metadata.FilterResolverMetadata;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.visitor.FilterInfoTypeVisitorFactory;
+
+public class ConditionalFilterResolverImpl implements FilterResolverIntf {
+
+ private static final long serialVersionUID = 1838955268462201691L;
+ protected Expression exp;
+ protected boolean isExpressionResolve;
+ protected boolean isIncludeFilter;
+ private DimColumnResolvedFilterInfo dimColResolvedFilterInfo;
+
+ public ConditionalFilterResolverImpl(Expression exp, boolean isExpressionResolve,
+ boolean isIncludeFilter) {
+ this.exp = exp;
+ this.isExpressionResolve = isExpressionResolve;
+ this.isIncludeFilter = isIncludeFilter;
+ this.dimColResolvedFilterInfo = new DimColumnResolvedFilterInfo();
+ }
+
+ /**
+ * This API will resolve the filter expression and generates the
+ * dictionaries for executing/evaluating the filter expressions in the
+ * executer layer.
+ *
+ * @throws FilterUnsupportedException
+ */
+ @Override public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier)
+ throws FilterUnsupportedException, IOException {
+ FilterResolverMetadata metadata = new FilterResolverMetadata();
+ metadata.setTableIdentifier(absoluteTableIdentifier);
+ if ((!isExpressionResolve) && exp instanceof BinaryConditionalExpression) {
+ BinaryConditionalExpression binaryConditionalExpression = (BinaryConditionalExpression) exp;
+ Expression leftExp = binaryConditionalExpression.getLeft();
+ Expression rightExp = binaryConditionalExpression.getRight();
+ if (leftExp instanceof ColumnExpression) {
+ ColumnExpression columnExpression = (ColumnExpression) leftExp;
+ metadata.setColumnExpression(columnExpression);
+ metadata.setExpression(rightExp);
+ metadata.setIncludeFilter(isIncludeFilter);
+ // If imei=imei comes in filter condition then we need to
+ // skip processing of right expression.
+ // This flow has reached here assuming that this is a single
+ // column expression.
+ // we need to check if the other expression contains column
+ // expression or not in depth.
+ CarbonDimension dimension = columnExpression.getDimension();
+ if (FilterUtil.checkIfExpressionContainsColumn(rightExp)
+ || FilterUtil.isExpressionNeedsToResolved(rightExp, isIncludeFilter) &&
+ dimension.hasEncoding(Encoding.DICTIONARY) && !dimension
+ .hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ isExpressionResolve = true;
+ } else {
+ //Visitor pattern is been used in this scenario inorder to populate the
+ // dimColResolvedFilterInfo
+ //visitable object with filter member values based on the visitor type, currently there
+ //3 types of visitors custom,direct and no dictionary, all types of visitor populate
+ //the visitable instance as per its buisness logic which is different for all the
+ // visitors.
+ dimColResolvedFilterInfo.populateFilterInfoBasedOnColumnType(
+ FilterInfoTypeVisitorFactory.getResolvedFilterInfoVisitor(columnExpression),
+ metadata);
+ }
+ } else if (rightExp instanceof ColumnExpression) {
+ ColumnExpression columnExpression = (ColumnExpression) rightExp;
+ metadata.setColumnExpression(columnExpression);
+ metadata.setExpression(leftExp);
+ metadata.setIncludeFilter(isIncludeFilter);
+ if (columnExpression.getDataType().equals(DataType.TIMESTAMP) ||
+ columnExpression.getDataType().equals(DataType.DATE)) {
+ isExpressionResolve = true;
+ } else {
+ // if imei=imei comes in filter condition then we need to
+ // skip processing of right expression.
+ // This flow has reached here assuming that this is a single
+ // column expression.
+ // we need to check if the other expression contains column
+ // expression or not in depth.
+ if (FilterUtil.checkIfExpressionContainsColumn(leftExp)) {
+ isExpressionResolve = true;
+ } else {
+
+ dimColResolvedFilterInfo.populateFilterInfoBasedOnColumnType(
+ FilterInfoTypeVisitorFactory.getResolvedFilterInfoVisitor(columnExpression),
+ metadata);
+
+ }
+ }
+ } else {
+ isExpressionResolve = true;
+ }
+ }
+ if (isExpressionResolve && exp instanceof ConditionalExpression) {
+ ConditionalExpression conditionalExpression = (ConditionalExpression) exp;
+ List<ColumnExpression> columnList = conditionalExpression.getColumnList();
+ metadata.setColumnExpression(columnList.get(0));
+ metadata.setExpression(exp);
+ metadata.setIncludeFilter(isIncludeFilter);
+ if (!columnList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY) ||
+ columnList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ dimColResolvedFilterInfo.populateFilterInfoBasedOnColumnType(
+ FilterInfoTypeVisitorFactory.getResolvedFilterInfoVisitor(columnList.get(0)), metadata);
+ } else if (columnList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY) &&
+ !(columnList.get(0).getDimension().getDataType() == DataType.STRUCT ||
+ columnList.get(0).getDimension().getDataType() == DataType.ARRAY)) {
+ dimColResolvedFilterInfo.setFilterValues(
+ FilterUtil.getFilterListForAllValues(absoluteTableIdentifier, exp, columnList.get(0),
+ isIncludeFilter));
+
+ dimColResolvedFilterInfo.setColumnIndex(columnList.get(0).getDimension().getOrdinal());
+ dimColResolvedFilterInfo.setDimension(columnList.get(0).getDimension());
+ }
+ }
+
+ }
+
+ /**
+ * Left node will not be presentin this scenario
+ *
+ * @return left node of type FilterResolverIntf instance
+ */
+ public FilterResolverIntf getLeft() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /**
+ * Right node will not be presentin this scenario
+ *
+ * @return left node of type FilterResolverIntf instance
+ */
+ @Override public FilterResolverIntf getRight() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ /**
+ * Method will return the DimColumnResolvedFilterInfo instance which consists
+ * the mapping of the respective dimension and its surrogates involved in
+ * filter expression.
+ *
+ * @return DimColumnResolvedFilterInfo
+ */
+ public DimColumnResolvedFilterInfo getDimColResolvedFilterInfo() {
+ return dimColResolvedFilterInfo;
+ }
+
+ /**
+ * method will calculates the start key based on the filter surrogates
+ */
+ public void getStartKey(long[] startKey,
+ SortedMap<Integer, byte[]> setOfStartKeyByteArray, List<long[]> startKeyList) {
+ FilterUtil.getStartKey(dimColResolvedFilterInfo.getDimensionResolvedFilterInstance(),
+ startKey, startKeyList);
+ FilterUtil.getStartKeyForNoDictionaryDimension(dimColResolvedFilterInfo,
+ setOfStartKeyByteArray);
+ }
+
+ /**
+ * get the start key based on the filter surrogates
+ */
+ @Override public void getEndKey(SegmentProperties segmentProperties, long[] endKeys,
+ SortedMap<Integer, byte[]> setOfEndKeyByteArray, List<long[]> endKeyList) {
+ FilterUtil.getEndKey(dimColResolvedFilterInfo.getDimensionResolvedFilterInstance(), endKeys,
+ segmentProperties, endKeyList);
+ FilterUtil.getEndKeyForNoDictionaryDimension(dimColResolvedFilterInfo,
+ setOfEndKeyByteArray);
+ }
+
+ /**
+ * Method will return the executer type for particular conditional resolver
+ * basically two types of executers will be formed for the conditional query.
+ *
+ * @return the filter executer type
+ */
+ @Override public FilterExecuterType getFilterExecuterType() {
+ switch (exp.getFilterExpressionType()) {
+ case NOT_EQUALS:
+ case NOT_IN:
+ return FilterExecuterType.EXCLUDE;
+
+ default:
+ return FilterExecuterType.INCLUDE;
+ }
+
+ }
+
+ @Override public Expression getFilterExpression() {
+ // TODO Auto-generated method stub
+ return exp;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java
new file mode 100644
index 0000000..b395e7c
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter.resolver;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.List;
+import java.util.SortedMap;
+
+import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+
+public interface FilterResolverIntf extends Serializable {
+
+ /**
+ * This API will resolve the filter expression and generates the
+ * dictionaries for executing/evaluating the filter expressions in the
+ * executer layer.
+ *
+ * @throws IOException
+ * @throws FilterUnsupportedException
+ */
+ void resolve(AbsoluteTableIdentifier absoluteTableIdentifier)
+ throws IOException, FilterUnsupportedException;
+
+ /**
+ * This API will provide the left column filter expression
+ * inorder to resolve the left expression filter.
+ *
+ * @return FilterResolverIntf
+ */
+ FilterResolverIntf getLeft();
+
+ /**
+ * API will provide the right column filter expression inorder to resolve
+ * the right expression filter.
+ *
+ * @return FilterResolverIntf
+ */
+ FilterResolverIntf getRight();
+
+ /**
+ * API will return the resolved filter instance, this instance will provide
+ * the resolved surrogates based on the applied filter
+ *
+ * @return DimColumnResolvedFilterInfo object
+ */
+ DimColumnResolvedFilterInfo getDimColResolvedFilterInfo();
+
+ /**
+ * API will get the start key based on the filter applied based on the key generator
+ *
+ * @param startKey
+ * @param setOfStartKeyByteArray
+ */
+ void getStartKey(long[] startKey, SortedMap<Integer, byte[]> setOfStartKeyByteArray,
+ List<long[]> startKeyList);
+
+ /**
+ * API will read the end key based on the max surrogate of
+ * particular dimension column
+ *
+ * @param endKeys
+ * @param setOfEndKeyByteArray
+ * @return
+ */
+ void getEndKey(SegmentProperties segmentProperties, long[] endKeys,
+ SortedMap<Integer, byte[]> setOfEndKeyByteArray, List<long[]> endKeyList);
+
+ /**
+ * API will return the filter executer type which will be used to evaluate
+ * the resolved filter while query execution
+ *
+ * @return FilterExecuterType.
+ */
+ FilterExecuterType getFilterExecuterType();
+
+ Expression getFilterExpression();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java
new file mode 100644
index 0000000..4ffb8ba
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.filter.resolver;
+
+import java.util.List;
+import java.util.SortedMap;
+
+import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.scan.expression.BinaryExpression;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+
+public class LogicalFilterResolverImpl implements FilterResolverIntf {
+ /**
+ *
+ */
+ private static final long serialVersionUID = 5734382980564402914L;
+
+ protected FilterResolverIntf leftEvalutor;
+
+ protected FilterResolverIntf rightEvalutor;
+
+ protected ExpressionType filterExpressionType;
+
+ private BinaryExpression filterExpression;
+
+ public LogicalFilterResolverImpl(FilterResolverIntf leftEvalutor,
+ FilterResolverIntf rightEvalutor, BinaryExpression currentExpression) {
+ this.leftEvalutor = leftEvalutor;
+ this.rightEvalutor = rightEvalutor;
+ this.filterExpressionType = currentExpression.getFilterExpressionType();
+ this.filterExpression = currentExpression;
+ }
+
+ /**
+ * Logical filter resolver will return the left and right filter expresison
+ * node for filter evaluation, so in this instance no implementation is required.
+ *
+ * @param absoluteTableIdentifier
+ */
+ @Override public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier) {
+
+ }
+
+ /**
+ * Since its a binary condition expresion the getLeft method will get the left
+ * node of filter expression
+ *
+ * @return FilterResolverIntf.
+ */
+ public FilterResolverIntf getLeft() {
+ return leftEvalutor;
+ }
+
+ /**
+ * Since its a binary condition expresion the getRight method will get the left
+ * node of filter expression
+ *
+ * @return FilterResolverIntf.
+ */
+ public FilterResolverIntf getRight() {
+ return rightEvalutor;
+ }
+
+ @Override public DimColumnResolvedFilterInfo getDimColResolvedFilterInfo() {
+ return null;
+ }
+
+ @Override
+ public void getStartKey(long[] startKey, SortedMap<Integer, byte[]> setOfStartKeyByteArray,
+ List<long[]> startKeyList) {
+
+ }
+
+ @Override public void getEndKey(SegmentProperties segmentProperties, long[] endKeys,
+ SortedMap<Integer, byte[]> setOfEndKeyByteArray, List<long[]> endKeyList) {
+
+ }
+
+ @Override public FilterExecuterType getFilterExecuterType() {
+ switch (filterExpressionType) {
+ case OR:
+ return FilterExecuterType.OR;
+ case AND:
+ return FilterExecuterType.AND;
+
+ default:
+ return null;
+ }
+ }
+
+ @Override public Expression getFilterExpression() {
+ return filterExpression;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
new file mode 100644
index 0000000..e2be44d
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.filter.resolver;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.conditional.ConditionalExpression;
+import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
+
+public class RowLevelFilterResolverImpl extends ConditionalFilterResolverImpl {
+
+ private static final long serialVersionUID = 176122729713729929L;
+
+ private List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList;
+ private List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList;
+ private AbsoluteTableIdentifier tableIdentifier;
+
+ public RowLevelFilterResolverImpl(Expression exp, boolean isExpressionResolve,
+ boolean isIncludeFilter, AbsoluteTableIdentifier tableIdentifier) {
+ super(exp, isExpressionResolve, isIncludeFilter);
+ dimColEvaluatorInfoList =
+ new ArrayList<DimColumnResolvedFilterInfo>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ msrColEvalutorInfoList = new ArrayList<MeasureColumnResolvedFilterInfo>(
+ CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ this.tableIdentifier = tableIdentifier;
+ }
+
+ /**
+ * Method which will resolve the filter expression by converting the filter member
+ * to its assigned dictionary values.
+ */
+ public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier) {
+ DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = null;
+ MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo = null;
+ int index = 0;
+ if (exp instanceof ConditionalExpression) {
+ ConditionalExpression conditionalExpression = (ConditionalExpression) exp;
+ List<ColumnExpression> columnList = conditionalExpression.getColumnList();
+ for (ColumnExpression columnExpression : columnList) {
+ if (columnExpression.isDimension()) {
+ dimColumnEvaluatorInfo = new DimColumnResolvedFilterInfo();
+ dimColumnEvaluatorInfo.setColumnIndex(columnExpression.getCarbonColumn().getOrdinal());
+ dimColumnEvaluatorInfo.setRowIndex(index++);
+ dimColumnEvaluatorInfo.setDimension(columnExpression.getDimension());
+ dimColumnEvaluatorInfo.setDimensionExistsInCurrentSilce(false);
+ dimColEvaluatorInfoList.add(dimColumnEvaluatorInfo);
+ } else {
+ msrColumnEvalutorInfo = new MeasureColumnResolvedFilterInfo();
+ msrColumnEvalutorInfo.setRowIndex(index++);
+ msrColumnEvalutorInfo
+ .setColumnIndex(columnExpression.getCarbonColumn().getOrdinal());
+ msrColumnEvalutorInfo.setType(columnExpression.getCarbonColumn().getDataType());
+ msrColEvalutorInfoList.add(msrColumnEvalutorInfo);
+ }
+ }
+ }
+ }
+
+ /**
+ * This method will provide the executer type to the callee inorder to identify
+ * the executer type for the filter resolution, Row level filter executer is a
+ * special executer since it get all the rows of the specified filter dimension
+ * and will be send to the spark for processing
+ */
+ @Override public FilterExecuterType getFilterExecuterType() {
+ return FilterExecuterType.ROWLEVEL;
+ }
+
+ /**
+ * Method will the read filter expression corresponding to the resolver.
+ * This method is required in row level executer inorder to evaluate the filter
+ * expression against spark, as mentioned above row level is a special type
+ * filter resolver.
+ *
+ * @return Expression
+ */
+ public Expression getFilterExpresion() {
+ return exp;
+ }
+
+ /**
+ * Method will return the DimColumnResolvedFilterInfo instance which consists
+ * the mapping of the respective dimension and its surrogates involved in
+ * filter expression.
+ *
+ * @return DimColumnResolvedFilterInfo
+ */
+ public List<DimColumnResolvedFilterInfo> getDimColEvaluatorInfoList() {
+ return dimColEvaluatorInfoList;
+ }
+
+ /**
+ * Method will return the DimColumnResolvedFilterInfo instance which containts
+ * measure level details.
+ *
+ * @return MeasureColumnResolvedFilterInfo
+ */
+ public List<MeasureColumnResolvedFilterInfo> getMsrColEvalutorInfoList() {
+ return msrColEvalutorInfoList;
+ }
+
+ /**
+ * Method will return table information which will be required for retrieving
+ * dictionary cache inorder to read all the members of respective dimension.
+ *
+ * @return AbsoluteTableIdentifier
+ */
+ public AbsoluteTableIdentifier getTableIdentifier() {
+ return tableIdentifier;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
new file mode 100644
index 0000000..90d5db7
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
@@ -0,0 +1,266 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.filter.resolver;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.SortedMap;
+
+import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.metadata.Encoding;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
+import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.conditional.BinaryConditionalExpression;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.expression.logical.BinaryLogicalExpression;
+import org.apache.carbondata.core.scan.filter.DimColumnFilterInfo;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
+
+public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverImpl {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 6629319265336666789L;
+ private List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList;
+ private List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList;
+ private AbsoluteTableIdentifier tableIdentifier;
+
+ public RowLevelRangeFilterResolverImpl(Expression exp, boolean isExpressionResolve,
+ boolean isIncludeFilter, AbsoluteTableIdentifier tableIdentifier) {
+ super(exp, isExpressionResolve, isIncludeFilter);
+ dimColEvaluatorInfoList =
+ new ArrayList<DimColumnResolvedFilterInfo>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ msrColEvalutorInfoList = new ArrayList<MeasureColumnResolvedFilterInfo>(
+ CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ this.tableIdentifier = tableIdentifier;
+ }
+
+ /**
+ * This method will return the filter values which is present in the range level
+ * conditional expressions.
+ *
+ * @return
+ */
+ public byte[][] getFilterRangeValues(SegmentProperties segmentProperties) {
+
+ if (null != dimColEvaluatorInfoList.get(0).getFilterValues() && !dimColEvaluatorInfoList.get(0)
+ .getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ List<byte[]> noDictFilterValuesList =
+ dimColEvaluatorInfoList.get(0).getFilterValues().getNoDictionaryFilterValuesList();
+ return noDictFilterValuesList.toArray((new byte[noDictFilterValuesList.size()][]));
+ } else if (null != dimColEvaluatorInfoList.get(0).getFilterValues() && dimColEvaluatorInfoList
+ .get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ return FilterUtil.getKeyArray(this.dimColEvaluatorInfoList.get(0).getFilterValues(),
+ this.dimColEvaluatorInfoList.get(0).getDimension(),
+ segmentProperties);
+ }
+ return null;
+
+ }
+
+ /**
+ * method will get the start key based on the filter surrogates
+ *
+ * @return start IndexKey
+ */
+ public void getStartKey(long[] startKey,
+ SortedMap<Integer, byte[]> noDictStartKeys, List<long[]> startKeyList) {
+ FilterUtil.getStartKey(dimColEvaluatorInfoList.get(0).getDimensionResolvedFilterInstance(),
+ startKey, startKeyList);
+ FilterUtil
+ .getStartKeyForNoDictionaryDimension(dimColEvaluatorInfoList.get(0), noDictStartKeys);
+ }
+
+ /**
+ * method will get the start key based on the filter surrogates
+ *
+ * @return end IndexKey
+ */
+ @Override public void getEndKey(SegmentProperties segmentProperties, long[] endKeys,
+ SortedMap<Integer, byte[]> noDicEndKeys, List<long[]> endKeyList) {
+ FilterUtil.getEndKey(dimColEvaluatorInfoList.get(0).getDimensionResolvedFilterInstance(),
+ endKeys, segmentProperties, endKeyList);
+ FilterUtil
+ .getEndKeyForNoDictionaryDimension(dimColEvaluatorInfoList.get(0), noDicEndKeys);
+ }
+
+ private List<byte[]> getNoDictionaryRangeValues() {
+ List<ExpressionResult> listOfExpressionResults = new ArrayList<ExpressionResult>(20);
+ if (this.getFilterExpression() instanceof BinaryConditionalExpression) {
+ listOfExpressionResults =
+ ((BinaryConditionalExpression) this.getFilterExpression()).getLiterals();
+ }
+ List<byte[]> filterValuesList = new ArrayList<byte[]>(20);
+ boolean invalidRowsPresent = false;
+ for (ExpressionResult result : listOfExpressionResults) {
+ try {
+ if (result.getString() == null) {
+ filterValuesList.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL.getBytes());
+ continue;
+ }
+ filterValuesList.add(result.getString().getBytes());
+ } catch (FilterIllegalMemberException e) {
+ // Any invalid member while evaluation shall be ignored, system will log the
+ // error only once since all rows the evaluation happens so inorder to avoid
+ // too much log inforation only once the log will be printed.
+ FilterUtil.logError(e, invalidRowsPresent);
+ }
+ }
+ Comparator<byte[]> filterNoDictValueComaparator = new Comparator<byte[]>() {
+ @Override public int compare(byte[] filterMember1, byte[] filterMember2) {
+ return ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterMember1, filterMember2);
+ }
+
+ };
+ Collections.sort(filterValuesList, filterNoDictValueComaparator);
+ return filterValuesList;
+ }
+
+ /**
+ * Method which will resolve the filter expression by converting the filter
+ * member to its assigned dictionary values.
+ */
+ public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier)
+ throws FilterUnsupportedException {
+ DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = null;
+ MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo = null;
+ int index = 0;
+ if (exp instanceof BinaryLogicalExpression) {
+ BinaryLogicalExpression conditionalExpression = (BinaryLogicalExpression) exp;
+ List<ColumnExpression> columnList = conditionalExpression.getColumnList();
+ for (ColumnExpression columnExpression : columnList) {
+ if (columnExpression.isDimension()) {
+ dimColumnEvaluatorInfo = new DimColumnResolvedFilterInfo();
+ DimColumnFilterInfo filterInfo = new DimColumnFilterInfo();
+ dimColumnEvaluatorInfo.setColumnIndex(columnExpression.getCarbonColumn().getOrdinal());
+ dimColumnEvaluatorInfo.setRowIndex(index++);
+ dimColumnEvaluatorInfo.setDimension(columnExpression.getDimension());
+ dimColumnEvaluatorInfo.setDimensionExistsInCurrentSilce(false);
+ if (columnExpression.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ filterInfo.setFilterList(getDirectSurrogateValues(columnExpression));
+ } else {
+ filterInfo.setFilterListForNoDictionaryCols(getNoDictionaryRangeValues());
+ }
+ filterInfo.setIncludeFilter(isIncludeFilter);
+ dimColumnEvaluatorInfo.setFilterValues(filterInfo);
+ dimColumnEvaluatorInfo
+ .addDimensionResolvedFilterInstance(columnExpression.getDimension(), filterInfo);
+ dimColEvaluatorInfoList.add(dimColumnEvaluatorInfo);
+ } else {
+ msrColumnEvalutorInfo = new MeasureColumnResolvedFilterInfo();
+ msrColumnEvalutorInfo.setRowIndex(index++);
+ msrColumnEvalutorInfo
+ .setColumnIndex(columnExpression.getCarbonColumn().getOrdinal());
+ msrColumnEvalutorInfo.setType(columnExpression.getCarbonColumn().getDataType());
+ msrColEvalutorInfoList.add(msrColumnEvalutorInfo);
+ }
+ }
+ }
+ }
+
+ private List<Integer> getDirectSurrogateValues(ColumnExpression columnExpression)
+ throws FilterUnsupportedException {
+ List<ExpressionResult> listOfExpressionResults = new ArrayList<ExpressionResult>(20);
+ DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
+ .getDirectDictionaryGenerator(columnExpression.getDimension().getDataType());
+
+ if (this.getFilterExpression() instanceof BinaryConditionalExpression) {
+ listOfExpressionResults =
+ ((BinaryConditionalExpression) this.getFilterExpression()).getLiterals();
+ }
+ List<Integer> filterValuesList = new ArrayList<Integer>(20);
+ try {
+ // if any filter member provided by user is invalid throw error else
+ // system can display inconsistent result.
+ for (ExpressionResult result : listOfExpressionResults) {
+ filterValuesList.add(directDictionaryGenerator
+ .generateDirectSurrogateKey(result.getString(),
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
+ }
+ } catch (FilterIllegalMemberException e) {
+ throw new FilterUnsupportedException(e);
+ }
+ return filterValuesList;
+ }
+
+ /**
+ * Method will return the DimColumnResolvedFilterInfo instance which consists
+ * the mapping of the respective dimension and its surrogates involved in
+ * filter expression.
+ *
+ * @return DimColumnResolvedFilterInfo
+ */
+ public List<DimColumnResolvedFilterInfo> getDimColEvaluatorInfoList() {
+ return dimColEvaluatorInfoList;
+ }
+
+ /**
+ * Method will return the DimColumnResolvedFilterInfo instance which containts
+ * measure level details.
+ *
+ * @return MeasureColumnResolvedFilterInfo
+ */
+ public List<MeasureColumnResolvedFilterInfo> getMsrColEvalutorInfoList() {
+ return msrColEvalutorInfoList;
+ }
+
+ public AbsoluteTableIdentifier getTableIdentifier() {
+ return tableIdentifier;
+ }
+
+ public Expression getFilterExpression() {
+ return this.exp;
+ }
+
+ /**
+ * This method will provide the executer type to the callee inorder to identify
+ * the executer type for the filter resolution, Row level filter executer is a
+ * special executer since it get all the rows of the specified filter dimension
+ * and will be send to the spark for processing
+ */
+ public FilterExecuterType getFilterExecuterType() {
+ switch (exp.getFilterExpressionType()) {
+ case GREATERTHAN:
+ return FilterExecuterType.ROWLEVEL_GREATERTHAN;
+ case GREATERTHAN_EQUALTO:
+ return FilterExecuterType.ROWLEVEL_GREATERTHAN_EQUALTO;
+ case LESSTHAN:
+ return FilterExecuterType.ROWLEVEL_LESSTHAN;
+ case LESSTHAN_EQUALTO:
+ return FilterExecuterType.ROWLEVEL_LESSTHAN_EQUALTO;
+
+ default:
+ return FilterExecuterType.ROWLEVEL;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/metadata/FilterResolverMetadata.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/metadata/FilterResolverMetadata.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/metadata/FilterResolverMetadata.java
new file mode 100644
index 0000000..779b532
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/metadata/FilterResolverMetadata.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter.resolver.metadata;
+
+import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.Expression;
+
+public class FilterResolverMetadata {
+ private AbsoluteTableIdentifier tableIdentifier;
+ private Expression expression;
+ private ColumnExpression columnExpression;
+ private boolean isIncludeFilter;
+
+ public AbsoluteTableIdentifier getTableIdentifier() {
+ return tableIdentifier;
+ }
+
+ public void setTableIdentifier(AbsoluteTableIdentifier tableIdentifier) {
+ this.tableIdentifier = tableIdentifier;
+ }
+
+ public Expression getExpression() {
+ return expression;
+ }
+
+ public void setExpression(Expression expression) {
+ this.expression = expression;
+ }
+
+ public ColumnExpression getColumnExpression() {
+ return columnExpression;
+ }
+
+ public void setColumnExpression(ColumnExpression columnExpression) {
+ this.columnExpression = columnExpression;
+ }
+
+ public boolean isIncludeFilter() {
+ return isIncludeFilter;
+ }
+
+ public void setIncludeFilter(boolean isIncludeFilter) {
+ this.isIncludeFilter = isIncludeFilter;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/DimColumnResolvedFilterInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/DimColumnResolvedFilterInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/DimColumnResolvedFilterInfo.java
new file mode 100644
index 0000000..78aabb8
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/DimColumnResolvedFilterInfo.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.filter.resolver.resolverinfo;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.DimColumnFilterInfo;
+import org.apache.carbondata.core.scan.filter.resolver.metadata.FilterResolverMetadata;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.visitor.ResolvedFilterInfoVisitorIntf;
+
+public class DimColumnResolvedFilterInfo implements Serializable {
+ /**
+ *
+ */
+ private static final long serialVersionUID = 3428115141211084114L;
+
+ /**
+ * column index in file
+ */
+ private int columnIndex = -1;
+
+ /**
+ * rowIndex
+ */
+ private int rowIndex = -1;
+
+ private boolean isDimensionExistsInCurrentSilce = true;
+
+ private String defaultValue;
+
+ private CarbonDimension dimension;
+
+ /**
+ * reolved filter object of a particlar filter Expression.
+ */
+ private DimColumnFilterInfo resolvedFilterValueObj;
+
+ private Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionResolvedFilter;
+
+ public DimColumnResolvedFilterInfo() {
+ dimensionResolvedFilter = new HashMap<CarbonDimension, List<DimColumnFilterInfo>>(20);
+ }
+
+ public void addDimensionResolvedFilterInstance(CarbonDimension dimension,
+ DimColumnFilterInfo filterResolvedObj) {
+ List<DimColumnFilterInfo> currentVals = dimensionResolvedFilter.get(dimension);
+ if (null == currentVals) {
+ currentVals = new ArrayList<DimColumnFilterInfo>(20);
+ currentVals.add(filterResolvedObj);
+ dimensionResolvedFilter.put(dimension, currentVals);
+ } else {
+ currentVals.add(filterResolvedObj);
+ }
+ }
+
+ public Map<CarbonDimension, List<DimColumnFilterInfo>> getDimensionResolvedFilterInstance() {
+ return dimensionResolvedFilter;
+ }
+
+ public CarbonDimension getDimension() {
+ return dimension;
+ }
+
+ public void setDimension(CarbonDimension dimension) {
+ this.dimension = dimension;
+ }
+
+ public int getColumnIndex() {
+ return columnIndex;
+ }
+
+ public void setColumnIndex(int columnIndex) {
+ this.columnIndex = columnIndex;
+ }
+
+ public DimColumnFilterInfo getFilterValues() {
+ return resolvedFilterValueObj;
+ }
+
+ public void setFilterValues(final DimColumnFilterInfo resolvedFilterValueObj) {
+ this.resolvedFilterValueObj = resolvedFilterValueObj;
+ }
+
+ public int getRowIndex() {
+ return rowIndex;
+ }
+
+ public void setRowIndex(int rowIndex) {
+ this.rowIndex = rowIndex;
+ }
+
+ public boolean isDimensionExistsInCurrentSilce() {
+ return isDimensionExistsInCurrentSilce;
+ }
+
+ public void setDimensionExistsInCurrentSilce(boolean isDimensionExistsInCurrentSilce) {
+ this.isDimensionExistsInCurrentSilce = isDimensionExistsInCurrentSilce;
+ }
+
+ public String getDefaultValue() {
+ return defaultValue;
+ }
+
+ public void setDefaultValue(String defaultValue) {
+ this.defaultValue = defaultValue;
+ }
+
+ public void populateFilterInfoBasedOnColumnType(ResolvedFilterInfoVisitorIntf visitor,
+ FilterResolverMetadata metadata) throws FilterUnsupportedException, IOException {
+ if (null != visitor) {
+ visitor.populateFilterResolvedInfo(this, metadata);
+ this.addDimensionResolvedFilterInstance(metadata.getColumnExpression().getDimension(),
+ this.getFilterValues());
+ this.setDimension(metadata.getColumnExpression().getDimension());
+ this.setColumnIndex(metadata.getColumnExpression().getDimension().getOrdinal());
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java
new file mode 100644
index 0000000..0cf2dce
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.filter.resolver.resolverinfo;
+
+import java.io.Serializable;
+
+import org.apache.carbondata.core.metadata.DataType;
+
+public class MeasureColumnResolvedFilterInfo implements Serializable {
+
+ private static final long serialVersionUID = 4222568289115151561L;
+
+ private int columnIndex = -1;
+
+ private int rowIndex = -1;
+
+ private DataType type;
+
+ public int getColumnIndex() {
+ return columnIndex;
+ }
+
+ public void setColumnIndex(int columnIndex) {
+ this.columnIndex = columnIndex;
+ }
+
+ public int getRowIndex() {
+ return rowIndex;
+ }
+
+ public void setRowIndex(int rowIndex) {
+ this.rowIndex = rowIndex;
+ }
+
+ public DataType getType() {
+ return type;
+ }
+
+ public void setType(DataType dataType) {
+ this.type = dataType;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
new file mode 100644
index 0000000..450674c
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter.resolver.resolverinfo.visitor;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.DimColumnFilterInfo;
+import org.apache.carbondata.core.scan.filter.resolver.metadata.FilterResolverMetadata;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+
+public class CustomTypeDictionaryVisitor implements ResolvedFilterInfoVisitorIntf {
+
+ /**
+ * This Visitor method is been used to resolve or populate the filter details
+ * by using custom type dictionary value, the filter membrers will be resolved using
+ * custom type function which will generate dictionary for the direct column type filter members
+ *
+ * @param visitableObj
+ * @param metadata
+ * @throws FilterUnsupportedException,if exception occurs while evaluating
+ * filter models.
+ */
+ public void populateFilterResolvedInfo(DimColumnResolvedFilterInfo visitableObj,
+ FilterResolverMetadata metadata) throws FilterUnsupportedException {
+ DimColumnFilterInfo resolvedFilterObject = null;
+
+ List<String> evaluateResultListFinal;
+ try {
+ evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString();
+ } catch (FilterIllegalMemberException e) {
+ throw new FilterUnsupportedException(e);
+ }
+ resolvedFilterObject = getDirectDictionaryValKeyMemberForFilter(metadata.getColumnExpression(),
+ evaluateResultListFinal, metadata.isIncludeFilter());
+ if (!metadata.isIncludeFilter() && null != resolvedFilterObject && !resolvedFilterObject
+ .getFilterList().contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) {
+ // Adding default surrogate key of null member inorder to not display the same while
+ // displaying the report as per hive compatibility.
+ resolvedFilterObject.getFilterList()
+ .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
+ Collections.sort(resolvedFilterObject.getFilterList());
+ }
+ visitableObj.setFilterValues(resolvedFilterObject);
+ }
+
+ private DimColumnFilterInfo getDirectDictionaryValKeyMemberForFilter(
+ ColumnExpression columnExpression, List<String> evaluateResultListFinal,
+ boolean isIncludeFilter) {
+ List<Integer> surrogates = new ArrayList<Integer>(20);
+ DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
+ .getDirectDictionaryGenerator(columnExpression.getDimension().getDataType());
+ // Reading the dictionary value direct
+ getSurrogateValuesForDictionary(evaluateResultListFinal, surrogates,
+ directDictionaryGenerator);
+
+ Collections.sort(surrogates);
+ DimColumnFilterInfo columnFilterInfo = null;
+ if (surrogates.size() > 0) {
+ columnFilterInfo = new DimColumnFilterInfo();
+ columnFilterInfo.setIncludeFilter(isIncludeFilter);
+ columnFilterInfo.setFilterList(surrogates);
+ }
+ return columnFilterInfo;
+ }
+
+ private void getSurrogateValuesForDictionary(List<String> evaluateResultListFinal,
+ List<Integer> surrogates, DirectDictionaryGenerator directDictionaryGenerator) {
+ String timeFormat = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT);
+ for (String filterMember : evaluateResultListFinal) {
+ surrogates
+ .add(directDictionaryGenerator.generateDirectSurrogateKey(filterMember, timeFormat));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
new file mode 100644
index 0000000..ddd8ab1
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter.resolver.resolverinfo.visitor;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.DimColumnFilterInfo;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.resolver.metadata.FilterResolverMetadata;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+
+public class DictionaryColumnVisitor implements ResolvedFilterInfoVisitorIntf {
+
+ /**
+ * This Visitor method is used to populate the visitableObj with direct dictionary filter details
+ * where the filters values will be resolve using dictionary cache.
+ *
+ * @param visitableObj
+ * @param metadata
+ * @throws FilterUnsupportedException,if exception occurs while evaluating
+ * filter models.
+ * @throws IOException
+ * @throws FilterUnsupportedException
+ */
+ public void populateFilterResolvedInfo(DimColumnResolvedFilterInfo visitableObj,
+ FilterResolverMetadata metadata) throws FilterUnsupportedException, IOException {
+ DimColumnFilterInfo resolvedFilterObject = null;
+ List<String> evaluateResultListFinal;
+ try {
+ evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString();
+ } catch (FilterIllegalMemberException e) {
+ throw new FilterUnsupportedException(e);
+ }
+ resolvedFilterObject = FilterUtil
+ .getFilterValues(metadata.getTableIdentifier(), metadata.getColumnExpression(),
+ evaluateResultListFinal, metadata.isIncludeFilter());
+ if (!metadata.isIncludeFilter() && null != resolvedFilterObject) {
+ // Adding default surrogate key of null member inorder to not display the same while
+ // displaying the report as per hive compatibility.
+ resolvedFilterObject.getFilterList()
+ .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
+ Collections.sort(resolvedFilterObject.getFilterList());
+ }
+ visitableObj.setFilterValues(resolvedFilterObject);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/FilterInfoTypeVisitorFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/FilterInfoTypeVisitorFactory.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/FilterInfoTypeVisitorFactory.java
new file mode 100644
index 0000000..a2c4aa8
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/FilterInfoTypeVisitorFactory.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter.resolver.resolverinfo.visitor;
+
+import org.apache.carbondata.core.metadata.Encoding;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+
+public class FilterInfoTypeVisitorFactory {
+
+ /**
+ * This factory method will be used in order to get the visitor instance based on the
+ * column expression metadata where filters has been applied.
+ *
+ * @param columnExpression
+ * @return
+ */
+ public static ResolvedFilterInfoVisitorIntf getResolvedFilterInfoVisitor(
+ ColumnExpression columnExpression) {
+ if (columnExpression.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ return new CustomTypeDictionaryVisitor();
+ } else if (!columnExpression.getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ return new NoDictionaryTypeVisitor();
+ } else if (columnExpression.getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ return new DictionaryColumnVisitor();
+ }
+
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
new file mode 100644
index 0000000..9174556
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter.resolver.resolverinfo.visitor;
+
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.DimColumnFilterInfo;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.resolver.metadata.FilterResolverMetadata;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+
+public class NoDictionaryTypeVisitor implements ResolvedFilterInfoVisitorIntf {
+
+ /**
+ * Visitor Method will update the filter related details in visitableObj, For no dictionary
+ * type columns the filter members will resolved directly, no need to look up in dictionary
+ * since it will not be part of dictionary, directly the actual data can be converted as
+ * byte[] and can be set. this type of encoding is effective when the particular column
+ * is having very high cardinality.
+ *
+ * @param visitableObj
+ * @param metadata
+ * @throws FilterUnsupportedException,if exception occurs while evaluating
+ * filter models.
+ */
+ public void populateFilterResolvedInfo(DimColumnResolvedFilterInfo visitableObj,
+ FilterResolverMetadata metadata) throws FilterUnsupportedException {
+ DimColumnFilterInfo resolvedFilterObject = null;
+ List<String> evaluateResultListFinal;
+ try {
+ evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString();
+ // Adding default null member inorder to not display the same while
+ // displaying the report as per hive compatibility.
+ if (!metadata.isIncludeFilter() && !evaluateResultListFinal
+ .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
+ evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
+ }
+ } catch (FilterIllegalMemberException e) {
+ throw new FilterUnsupportedException(e);
+ }
+ resolvedFilterObject = FilterUtil
+ .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter());
+ visitableObj.setFilterValues(resolvedFilterObject);
+ }
+}
[26/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
new file mode 100644
index 0000000..7b29c8b
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
@@ -0,0 +1,604 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.executor.infos;
+
+import java.util.Map;
+
+import org.apache.carbondata.core.cache.dictionary.Dictionary;
+import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.datastore.IndexKey;
+import org.apache.carbondata.core.datastore.block.AbstractIndex;
+import org.apache.carbondata.core.keygenerator.KeyGenerator;
+import org.apache.carbondata.core.scan.filter.GenericQueryType;
+import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
+import org.apache.carbondata.core.scan.model.QueryDimension;
+import org.apache.carbondata.core.scan.model.QueryMeasure;
+
+/**
+ * Below class will have all the properties which needed during query execution
+ * for one block
+ */
+public class BlockExecutionInfo {
+
+ /**
+ * block on which query will be executed
+ */
+ private AbstractIndex blockIndex;
+
+ /**
+ * each segment key size can be different and in that case we need to update
+ * the fixed key with latest segment key generator. so this property will
+ * tell whether this is required or not if key size is same then it is not
+ * required
+ */
+ private boolean isFixedKeyUpdateRequired;
+
+ /**
+ * below to store all the information required for aggregation during query
+ * execution
+ */
+ private AggregatorInfo aggregatorInfo;
+
+ /**
+ * this will be used to get the first tentative block from which query
+ * execution start, this will be useful in case of filter query to get the
+ * start block based on filter values
+ */
+ private IndexKey startKey;
+
+ /**
+ * this will be used to get the last tentative block till which scanning
+ * will be done, this will be useful in case of filter query to get the last
+ * block based on filter values
+ */
+ private IndexKey endKey;
+
+ private String blockId;
+
+ /**
+ * masked byte for block which will be used to unpack the fixed length key,
+ * this will be used for updating the older block key with new block key
+ * generator
+ */
+ private int[] maskedByteForBlock;
+
+ /**
+ * total number of dimension in block
+ */
+ private int totalNumberDimensionBlock;
+
+ /**
+ * total number of measure in block
+ */
+ private int totalNumberOfMeasureBlock;
+
+ /**
+ * will be used to read the dimension block from file
+ */
+ private int[][] allSelectedDimensionBlocksIndexes;
+
+ /**
+ * will be used to read the measure block from file
+ */
+ private int[][] allSelectedMeasureBlocksIndexes;
+
+ /**
+ * this will be used to update the older block fixed length keys with the
+ * new block fixed length key
+ */
+ private KeyStructureInfo keyStructureInfo;
+
+ /**
+ * first block from which query execution will start
+ */
+ private DataRefNode firstDataBlock;
+
+ /**
+ * number of block to be scanned in the query
+ */
+ private long numberOfBlockToScan;
+
+ /**
+ * key size of the fixed length dimension column
+ */
+ private int fixedLengthKeySize;
+
+ /**
+ * dictionary column block indexes based on query
+ */
+ private int[] dictionaryColumnBlockIndex;
+ /**
+ * no dictionary column block indexes in based on the query order
+ */
+ private int[] noDictionaryBlockIndexes;
+
+ /**
+ * key generator used for generating the table block fixed length key
+ */
+ private KeyGenerator blockKeyGenerator;
+
+ /**
+ * each column value size
+ */
+ private int[] eachColumnValueSize;
+
+ /**
+ * column group block index in file to key structure info mapping
+ */
+ private Map<Integer, KeyStructureInfo> columnGroupToKeyStructureInfo;
+
+ /**
+ * mapping of dictionary dimension to its dictionary mapping which will be
+ * used to get the actual data from dictionary for aggregation, sorting
+ */
+ private Map<String, Dictionary> columnIdToDcitionaryMapping;
+
+ /**
+ * filter tree to execute the filter
+ */
+ private FilterExecuter filterExecuterTree;
+
+ /**
+ * whether it needs only raw byte records with out aggregation.
+ */
+ private boolean isRawRecordDetailQuery;
+
+ /**
+ * start index of blocklets
+ */
+ private int startBlockletIndex;
+
+ /**
+ * number of blocklet to be scanned
+ */
+ private int numberOfBlockletToScan;
+
+ /**
+ * complexParentIndexToQueryMap
+ */
+ private Map<Integer, GenericQueryType> complexParentIndexToQueryMap;
+
+ /**
+ * complex dimension parent block indexes;
+ */
+ private int[] complexColumnParentBlockIndexes;
+
+ /**
+ * @return the tableBlock
+ */
+ public AbstractIndex getDataBlock() {
+ return blockIndex;
+ }
+
+ /**
+ * list of dimension selected for in query
+ */
+ private QueryDimension[] queryDimensions;
+
+ /**
+ * list of measure selected in query
+ */
+ private QueryMeasure[] queryMeasures;
+
+ /**
+ * whether it needs to read data in vector/columnar format.
+ */
+ private boolean vectorBatchCollector;
+
+ /**
+ * absolute table identifier
+ */
+ private AbsoluteTableIdentifier absoluteTableIdentifier;
+
+ public AbsoluteTableIdentifier getAbsoluteTableIdentifier() {
+ return absoluteTableIdentifier;
+ }
+
+ public void setAbsoluteTableIdentifier(AbsoluteTableIdentifier absoluteTableIdentifier) {
+ this.absoluteTableIdentifier = absoluteTableIdentifier;
+ }
+
+ /**
+ * @param blockIndex the tableBlock to set
+ */
+ public void setDataBlock(AbstractIndex blockIndex) {
+ this.blockIndex = blockIndex;
+ }
+
+ /**
+ * @return the isFixedKeyUpdateRequired
+ */
+ public boolean isFixedKeyUpdateRequired() {
+ return isFixedKeyUpdateRequired;
+ }
+
+ /**
+ * @param isFixedKeyUpdateRequired the isFixedKeyUpdateRequired to set
+ */
+ public void setFixedKeyUpdateRequired(boolean isFixedKeyUpdateRequired) {
+ this.isFixedKeyUpdateRequired = isFixedKeyUpdateRequired;
+ }
+
+ /**
+ * @return the aggregatorInfos
+ */
+ public AggregatorInfo getAggregatorInfo() {
+ return aggregatorInfo;
+ }
+
+ /**
+ * @param aggregatorInfo the aggregatorInfos to set
+ */
+ public void setAggregatorInfo(AggregatorInfo aggregatorInfo) {
+ this.aggregatorInfo = aggregatorInfo;
+ }
+
+ /**
+ * @return the startKey
+ */
+ public IndexKey getStartKey() {
+ return startKey;
+ }
+
+ /**
+ * @param startKey the startKey to set
+ */
+ public void setStartKey(IndexKey startKey) {
+ this.startKey = startKey;
+ }
+
+ /**
+ * @return the endKey
+ */
+ public IndexKey getEndKey() {
+ return endKey;
+ }
+
+ /**
+ * @param endKey the endKey to set
+ */
+ public void setEndKey(IndexKey endKey) {
+ this.endKey = endKey;
+ }
+
+ /**
+ * @return the maskedByteForBlock
+ */
+ public int[] getMaskedByteForBlock() {
+ return maskedByteForBlock;
+ }
+
+
+
+ /**
+ * @param maskedByteForBlock the maskedByteForBlock to set
+ */
+ public void setMaskedByteForBlock(int[] maskedByteForBlock) {
+ this.maskedByteForBlock = maskedByteForBlock;
+ }
+
+ /**
+ * @return the totalNumberDimensionBlock
+ */
+ public int getTotalNumberDimensionBlock() {
+ return totalNumberDimensionBlock;
+ }
+
+ /**
+ * @param totalNumberDimensionBlock the totalNumberDimensionBlock to set
+ */
+ public void setTotalNumberDimensionBlock(int totalNumberDimensionBlock) {
+ this.totalNumberDimensionBlock = totalNumberDimensionBlock;
+ }
+
+ /**
+ * @return the totalNumberOfMeasureBlock
+ */
+ public int getTotalNumberOfMeasureBlock() {
+ return totalNumberOfMeasureBlock;
+ }
+
+ /**
+ * @param totalNumberOfMeasureBlock the totalNumberOfMeasureBlock to set
+ */
+ public void setTotalNumberOfMeasureBlock(int totalNumberOfMeasureBlock) {
+ this.totalNumberOfMeasureBlock = totalNumberOfMeasureBlock;
+ }
+
+ /**
+ * @return the allSelectedDimensionBlocksIndexes
+ */
+ public int[][] getAllSelectedDimensionBlocksIndexes() {
+ return allSelectedDimensionBlocksIndexes;
+ }
+
+ /**
+ * @param allSelectedDimensionBlocksIndexes the allSelectedDimensionBlocksIndexes to set
+ */
+ public void setAllSelectedDimensionBlocksIndexes(int[][] allSelectedDimensionBlocksIndexes) {
+ this.allSelectedDimensionBlocksIndexes = allSelectedDimensionBlocksIndexes;
+ }
+
+ /**
+ * @return the allSelectedMeasureBlocksIndexes
+ */
+ public int[][] getAllSelectedMeasureBlocksIndexes() {
+ return allSelectedMeasureBlocksIndexes;
+ }
+
+ /**
+ * @param allSelectedMeasureBlocksIndexes the allSelectedMeasureBlocksIndexes to set
+ */
+ public void setAllSelectedMeasureBlocksIndexes(int[][] allSelectedMeasureBlocksIndexes) {
+ this.allSelectedMeasureBlocksIndexes = allSelectedMeasureBlocksIndexes;
+ }
+
+ /**
+ * @return the restructureInfos
+ */
+ public KeyStructureInfo getKeyStructureInfo() {
+ return keyStructureInfo;
+ }
+
+ /**
+ * @param keyStructureInfo the restructureInfos to set
+ */
+ public void setKeyStructureInfo(KeyStructureInfo keyStructureInfo) {
+ this.keyStructureInfo = keyStructureInfo;
+ }
+
+ /**
+ * @return the firstDataBlock
+ */
+ public DataRefNode getFirstDataBlock() {
+ return firstDataBlock;
+ }
+
+ /**
+ * @param firstDataBlock the firstDataBlock to set
+ */
+ public void setFirstDataBlock(DataRefNode firstDataBlock) {
+ this.firstDataBlock = firstDataBlock;
+ }
+
+ /**
+ * @return the numberOfBlockToScan
+ */
+ public long getNumberOfBlockToScan() {
+ return numberOfBlockToScan;
+ }
+
+ /**
+ * @param numberOfBlockToScan the numberOfBlockToScan to set
+ */
+ public void setNumberOfBlockToScan(long numberOfBlockToScan) {
+ this.numberOfBlockToScan = numberOfBlockToScan;
+ }
+
+ /**
+ * @return the fixedLengthKeySize
+ */
+ public int getFixedLengthKeySize() {
+ return fixedLengthKeySize;
+ }
+
+ /**
+ * @param fixedLengthKeySize the fixedLengthKeySize to set
+ */
+ public void setFixedLengthKeySize(int fixedLengthKeySize) {
+ this.fixedLengthKeySize = fixedLengthKeySize;
+ }
+
+ /**
+ * @return the filterEvaluatorTree
+ */
+ public FilterExecuter getFilterExecuterTree() {
+ return filterExecuterTree;
+ }
+
+ /**
+ * @param filterExecuterTree the filterEvaluatorTree to set
+ */
+ public void setFilterExecuterTree(FilterExecuter filterExecuterTree) {
+ this.filterExecuterTree = filterExecuterTree;
+ }
+
+ /**
+ * @return the tableBlockKeyGenerator
+ */
+ public KeyGenerator getBlockKeyGenerator() {
+ return blockKeyGenerator;
+ }
+
+ /**
+ * @param tableBlockKeyGenerator the tableBlockKeyGenerator to set
+ */
+ public void setBlockKeyGenerator(KeyGenerator tableBlockKeyGenerator) {
+ this.blockKeyGenerator = tableBlockKeyGenerator;
+ }
+
+ /**
+ * @return the eachColumnValueSize
+ */
+ public int[] getEachColumnValueSize() {
+ return eachColumnValueSize;
+ }
+
+ /**
+ * @param eachColumnValueSize the eachColumnValueSize to set
+ */
+ public void setEachColumnValueSize(int[] eachColumnValueSize) {
+ this.eachColumnValueSize = eachColumnValueSize;
+ }
+
+ /**
+ * @return the dictionaryColumnBlockIndex
+ */
+ public int[] getDictionaryColumnBlockIndex() {
+ return dictionaryColumnBlockIndex;
+ }
+
+ /**
+ * @param dictionaryColumnBlockIndex the dictionaryColumnBlockIndex to set
+ */
+ public void setDictionaryColumnBlockIndex(int[] dictionaryColumnBlockIndex) {
+ this.dictionaryColumnBlockIndex = dictionaryColumnBlockIndex;
+ }
+
+ /**
+ * @return the noDictionaryBlockIndexes
+ */
+ public int[] getNoDictionaryBlockIndexes() {
+ return noDictionaryBlockIndexes;
+ }
+
+ /**
+ * @param noDictionaryBlockIndexes the noDictionaryBlockIndexes to set
+ */
+ public void setNoDictionaryBlockIndexes(int[] noDictionaryBlockIndexes) {
+ this.noDictionaryBlockIndexes = noDictionaryBlockIndexes;
+ }
+
+ /**
+ * @return the columnGroupToKeyStructureInfo
+ */
+ public Map<Integer, KeyStructureInfo> getColumnGroupToKeyStructureInfo() {
+ return columnGroupToKeyStructureInfo;
+ }
+
+ /**
+ * @param columnGroupToKeyStructureInfo the columnGroupToKeyStructureInfo to set
+ */
+ public void setColumnGroupToKeyStructureInfo(
+ Map<Integer, KeyStructureInfo> columnGroupToKeyStructureInfo) {
+ this.columnGroupToKeyStructureInfo = columnGroupToKeyStructureInfo;
+ }
+
+ /**
+ * @param columnIdToDcitionaryMapping the columnIdToDcitionaryMapping to set
+ */
+ public void setColumnIdToDcitionaryMapping(Map<String, Dictionary> columnIdToDcitionaryMapping) {
+ this.columnIdToDcitionaryMapping = columnIdToDcitionaryMapping;
+ }
+
+ public boolean isRawRecordDetailQuery() {
+ return isRawRecordDetailQuery;
+ }
+
+ public void setRawRecordDetailQuery(boolean rawRecordDetailQuery) {
+ isRawRecordDetailQuery = rawRecordDetailQuery;
+ }
+
+ /**
+ * @return the complexParentIndexToQueryMap
+ */
+ public Map<Integer, GenericQueryType> getComlexDimensionInfoMap() {
+ return complexParentIndexToQueryMap;
+ }
+
+ /**
+ * @param complexDimensionInfoMap the complexParentIndexToQueryMap to set
+ */
+ public void setComplexDimensionInfoMap(Map<Integer, GenericQueryType> complexDimensionInfoMap) {
+ this.complexParentIndexToQueryMap = complexDimensionInfoMap;
+ }
+
+ /**
+ * @return the complexColumnParentBlockIndexes
+ */
+ public int[] getComplexColumnParentBlockIndexes() {
+ return complexColumnParentBlockIndexes;
+ }
+
+ /**
+ * @param complexColumnParentBlockIndexes the complexColumnParentBlockIndexes to set
+ */
+ public void setComplexColumnParentBlockIndexes(int[] complexColumnParentBlockIndexes) {
+ this.complexColumnParentBlockIndexes = complexColumnParentBlockIndexes;
+ }
+
+ public QueryDimension[] getQueryDimensions() {
+ return queryDimensions;
+ }
+
+ public void setQueryDimensions(QueryDimension[] queryDimensions) {
+ this.queryDimensions = queryDimensions;
+ }
+
+ public QueryMeasure[] getQueryMeasures() {
+ return queryMeasures;
+ }
+
+ public void setQueryMeasures(QueryMeasure[] queryMeasures) {
+ this.queryMeasures = queryMeasures;
+ }
+
+ /**
+ * The method to set the number of blocklets to be scanned
+ *
+ * @param numberOfBlockletToScan
+ */
+ public void setNumberOfBlockletToScan(int numberOfBlockletToScan) {
+ this.numberOfBlockletToScan = numberOfBlockletToScan;
+ }
+
+ /**
+ * get the no of blocklet to be scanned
+ *
+ * @return
+ */
+ public int getNumberOfBlockletToScan() {
+ return numberOfBlockletToScan;
+ }
+
+ /**
+ * returns the blocklet index to be scanned
+ *
+ * @return
+ */
+ public int getStartBlockletIndex() {
+ return startBlockletIndex;
+ }
+
+ /**
+ * set the blocklet index to be scanned
+ *
+ * @param startBlockletIndex
+ */
+ public void setStartBlockletIndex(int startBlockletIndex) {
+ this.startBlockletIndex = startBlockletIndex;
+ }
+
+ public boolean isVectorBatchCollector() {
+ return vectorBatchCollector;
+ }
+
+ public void setVectorBatchCollector(boolean vectorBatchCollector) {
+ this.vectorBatchCollector = vectorBatchCollector;
+ }
+
+ public String getBlockId() {
+ return blockId;
+ }
+
+ public void setBlockId(String blockId) {
+ this.blockId = blockId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/KeyStructureInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/KeyStructureInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/KeyStructureInfo.java
new file mode 100644
index 0000000..ad45eb8
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/KeyStructureInfo.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.executor.infos;
+
+import org.apache.carbondata.core.keygenerator.KeyGenerator;
+
+/**
+ * Below class will store the structure of the key
+ * used during query execution
+ */
+public class KeyStructureInfo {
+
+ /**
+ * it's actually a latest key generator
+ * last table block as this key generator will be used to
+ * to update the mdkey of the older slice with the new slice
+ */
+ private KeyGenerator keyGenerator;
+
+ /**
+ * mask bytes ranges for the query
+ */
+ private int[] maskByteRanges;
+
+ /**
+ * max key for query execution
+ */
+ private byte[] maxKey;
+
+ /**
+ * dimension ordinals inside the column group
+ */
+ private int[] mdkeyQueryDimensionOrdinal;
+
+ /**
+ * @return the keyGenerator
+ */
+ public KeyGenerator getKeyGenerator() {
+ return keyGenerator;
+ }
+
+ /**
+ * @param keyGenerator the keyGenerator to set
+ */
+ public void setKeyGenerator(KeyGenerator keyGenerator) {
+ this.keyGenerator = keyGenerator;
+ }
+
+ /**
+ * @return the maskByteRanges
+ */
+ public int[] getMaskByteRanges() {
+ return maskByteRanges;
+ }
+
+ /**
+ * @param maskByteRanges the maskByteRanges to set
+ */
+ public void setMaskByteRanges(int[] maskByteRanges) {
+ this.maskByteRanges = maskByteRanges;
+ }
+
+ /**
+ * @return the maxKey
+ */
+ public byte[] getMaxKey() {
+ return maxKey;
+ }
+
+ /**
+ * @param maxKey the maxKey to set
+ */
+ public void setMaxKey(byte[] maxKey) {
+ this.maxKey = maxKey;
+ }
+
+ public int[] getMdkeyQueryDimensionOrdinal() {
+ return mdkeyQueryDimensionOrdinal;
+ }
+
+ public void setMdkeyQueryDimensionOrdinal(int[] mdkeyQueryDimensionOrdinal) {
+ this.mdkeyQueryDimensionOrdinal = mdkeyQueryDimensionOrdinal;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
new file mode 100644
index 0000000..676d978
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
@@ -0,0 +1,882 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.executor.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.carbondata.core.cache.Cache;
+import org.apache.carbondata.core.cache.CacheProvider;
+import org.apache.carbondata.core.cache.CacheType;
+import org.apache.carbondata.core.cache.dictionary.Dictionary;
+import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
+import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.metadata.CarbonMetadata;
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.metadata.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.keygenerator.KeyGenerator;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.scan.complextypes.ArrayQueryType;
+import org.apache.carbondata.core.scan.complextypes.PrimitiveQueryType;
+import org.apache.carbondata.core.scan.complextypes.StructQueryType;
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.filter.GenericQueryType;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.model.QueryDimension;
+import org.apache.carbondata.core.scan.model.QueryMeasure;
+import org.apache.carbondata.core.scan.model.QueryModel;
+
+import org.apache.commons.lang3.ArrayUtils;
+
+/**
+ * Utility class for query execution
+ */
+public class QueryUtil {
+
+ /**
+ * Below method will be used to get the masked byte range based on the query
+ * dimension. It will give the range in the mdkey. This will be used to get
+ * the actual key array from masked mdkey
+ *
+ * @param queryDimensions query dimension selected in query
+ * @param keyGenerator key generator
+ * @return masked key
+ */
+ public static int[] getMaskedByteRange(List<QueryDimension> queryDimensions,
+ KeyGenerator keyGenerator) {
+ Set<Integer> byteRangeSet = new TreeSet<Integer>();
+ int[] byteRange = null;
+ for (int i = 0; i < queryDimensions.size(); i++) {
+
+ // as no dictionary column and complex type columns
+ // are not selected in the mdkey
+ // so we will not select the those dimension for calculating the
+ // range
+ if (queryDimensions.get(i).getDimension().getKeyOrdinal() == -1) {
+ continue;
+ }
+ // get the offset of the dimension in the mdkey
+ byteRange =
+ keyGenerator.getKeyByteOffsets(queryDimensions.get(i).getDimension().getKeyOrdinal());
+ for (int j = byteRange[0]; j <= byteRange[1]; j++) {
+ byteRangeSet.add(j);
+ }
+ }
+ int[] maksedByteRange = new int[byteRangeSet.size()];
+ int index = 0;
+ Iterator<Integer> iterator = byteRangeSet.iterator();
+ // add the masked byte range
+ while (iterator.hasNext()) {
+ maksedByteRange[index++] = iterator.next();
+ }
+ return maksedByteRange;
+ }
+
+ public static int[] getMaskedByteRangeBasedOrdinal(List<Integer> ordinals,
+ KeyGenerator keyGenerator) {
+ Set<Integer> byteRangeSet = new TreeSet<Integer>();
+ int[] byteRange = null;
+ for (int i = 0; i < ordinals.size(); i++) {
+
+ // get the offset of the dimension in the mdkey
+ byteRange = keyGenerator.getKeyByteOffsets(ordinals.get(i));
+ for (int j = byteRange[0]; j <= byteRange[1]; j++) {
+ byteRangeSet.add(j);
+ }
+ }
+ int[] maksedByteRange = new int[byteRangeSet.size()];
+ int index = 0;
+ Iterator<Integer> iterator = byteRangeSet.iterator();
+ // add the masked byte range
+ while (iterator.hasNext()) {
+ maksedByteRange[index++] = iterator.next();
+ }
+ return maksedByteRange;
+ }
+
+ /**
+ * Below method will return the max key based on the dimension ordinal
+ *
+ * @param keyOrdinalList
+ * @param generator
+ * @return
+ * @throws KeyGenException
+ */
+ public static byte[] getMaxKeyBasedOnOrinal(List<Integer> keyOrdinalList, KeyGenerator generator)
+ throws KeyGenException {
+ long[] max = new long[generator.getDimCount()];
+ Arrays.fill(max, 0L);
+
+ for (int i = 0; i < keyOrdinalList.size(); i++) {
+ // adding for dimension which is selected in query
+ max[keyOrdinalList.get(i)] = Long.MAX_VALUE;
+ }
+ return generator.generateKey(max);
+ }
+
+ /**
+ * To get the max key based on dimensions. i.e. all other dimensions will be
+ * set to 0 bits and the required query dimension will be masked with all
+ * LONG.MAX so that we can mask key and then compare while aggregating This
+ * can be useful during filter query when only few dimensions were selected
+ * out of row group
+ *
+ * @param queryDimensions dimension selected in query
+ * @param generator key generator
+ * @return max key for dimension
+ * @throws KeyGenException if any problem while generating the key
+ */
+ public static byte[] getMaxKeyBasedOnDimensions(List<QueryDimension> queryDimensions,
+ KeyGenerator generator) throws KeyGenException {
+ long[] max = new long[generator.getDimCount()];
+ Arrays.fill(max, 0L);
+
+ for (int i = 0; i < queryDimensions.size(); i++) {
+ // as no dictionary column and complex type columns
+ // are not selected in the mdkey
+ // so we will not select the those dimension for calculating the
+ // range
+ if (queryDimensions.get(i).getDimension().getKeyOrdinal() == -1) {
+ continue;
+ }
+ // adding for dimension which is selected in query
+ max[queryDimensions.get(i).getDimension().getKeyOrdinal()] = Long.MAX_VALUE;
+ }
+
+ return generator.generateKey(max);
+ }
+
+ /**
+ * Below method will be used to get the masked key for query
+ *
+ * @param keySize size of the masked key
+ * @param maskedKeyRanges masked byte range
+ * @return masked bytes
+ */
+ public static int[] getMaskedByte(int keySize, int[] maskedKeyRanges) {
+ int[] maskedKey = new int[keySize];
+ // all the non selected dimension will be filled with -1
+ Arrays.fill(maskedKey, -1);
+ for (int i = 0; i < maskedKeyRanges.length; i++) {
+ maskedKey[maskedKeyRanges[i]] = i;
+ }
+ return maskedKey;
+ }
+
+ /**
+ * Below method will be used to get the dimension block index in file based
+ * on query dimension
+ *
+ * @param queryDimensions query dimension
+ * @param dimensionOrdinalToBlockMapping mapping of dimension block in file to query dimension
+ * @return block index of file
+ */
+ public static int[] getDimensionsBlockIndexes(List<QueryDimension> queryDimensions,
+ Map<Integer, Integer> dimensionOrdinalToBlockMapping,
+ List<CarbonDimension> customAggregationDimension, Set<CarbonDimension> filterDimensions,
+ List<Integer> allProjectionListDimensionIndexes) {
+ // using set as in row group columns will point to same block
+ Set<Integer> dimensionBlockIndex = new HashSet<Integer>();
+ Set<Integer> filterDimensionOrdinal = getFilterDimensionOrdinal(filterDimensions);
+ int blockIndex = 0;
+ for (int i = 0; i < queryDimensions.size(); i++) {
+ if (queryDimensions.get(i).getDimension().hasEncoding(Encoding.IMPLICIT)) {
+ continue;
+ }
+ allProjectionListDimensionIndexes.add(queryDimensions.get(i).getDimension().getOrdinal());
+
+ if (!filterDimensionOrdinal.contains(queryDimensions.get(i).getDimension().getOrdinal())) {
+ blockIndex =
+ dimensionOrdinalToBlockMapping.get(queryDimensions.get(i).getDimension().getOrdinal());
+ dimensionBlockIndex.add(blockIndex);
+ if (queryDimensions.get(i).getDimension().numberOfChild() > 0) {
+ addChildrenBlockIndex(dimensionBlockIndex, queryDimensions.get(i).getDimension());
+ }
+ }
+ }
+ for (int i = 0; i < customAggregationDimension.size(); i++) {
+ blockIndex =
+ dimensionOrdinalToBlockMapping.get(customAggregationDimension.get(i).getOrdinal());
+ // not adding the children dimension as dimension aggregation
+ // is not push down in case of complex dimension
+ dimensionBlockIndex.add(blockIndex);
+ }
+ int[] dimensionIndex = ArrayUtils
+ .toPrimitive(dimensionBlockIndex.toArray(new Integer[dimensionBlockIndex.size()]));
+ Arrays.sort(dimensionIndex);
+ return dimensionIndex;
+ }
+
+ /**
+ * Below method will be used to add the children block index
+ * this will be basically for complex dimension which will have children
+ *
+ * @param blockIndexes block indexes
+ * @param dimension parent dimension
+ */
+ private static void addChildrenBlockIndex(Set<Integer> blockIndexes, CarbonDimension dimension) {
+ for (int i = 0; i < dimension.numberOfChild(); i++) {
+ addChildrenBlockIndex(blockIndexes, dimension.getListOfChildDimensions().get(i));
+ blockIndexes.add(dimension.getListOfChildDimensions().get(i).getOrdinal());
+ }
+ }
+
+ /**
+ * Below method will be used to get the dictionary mapping for all the
+ * dictionary encoded dimension present in the query
+ *
+ * @param queryDimensions query dimension present in the query this will be used to
+ * convert the result from surrogate key to actual data
+ * @param absoluteTableIdentifier absolute table identifier
+ * @return dimension unique id to its dictionary map
+ * @throws IOException
+ */
+ public static Map<String, Dictionary> getDimensionDictionaryDetail(
+ List<QueryDimension> queryDimensions, Set<CarbonDimension> filterComplexDimensions,
+ AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException {
+ // to store dimension unique column id list, this is required as
+ // dimension can be present in
+ // query dimension, as well as some aggregation function will be applied
+ // in the same dimension
+ // so we need to get only one instance of dictionary
+ // direct dictionary skip is done only for the dictionary lookup
+ Set<String> dictionaryDimensionFromQuery = new HashSet<String>();
+ for (int i = 0; i < queryDimensions.size(); i++) {
+ List<Encoding> encodingList = queryDimensions.get(i).getDimension().getEncoder();
+ // TODO need to remove the data type check for parent column in complex type no need to
+ // write encoding dictionary
+ if (CarbonUtil.hasEncoding(encodingList, Encoding.DICTIONARY) && !CarbonUtil
+ .hasEncoding(encodingList, Encoding.DIRECT_DICTIONARY) && !CarbonUtil
+ .hasEncoding(encodingList, Encoding.IMPLICIT)) {
+
+ if (queryDimensions.get(i).getDimension().numberOfChild() == 0) {
+ dictionaryDimensionFromQuery.add(queryDimensions.get(i).getDimension().getColumnId());
+ }
+ if (queryDimensions.get(i).getDimension().numberOfChild() > 0) {
+ getChildDimensionDictionaryDetail(queryDimensions.get(i).getDimension(),
+ dictionaryDimensionFromQuery);
+ }
+ }
+ }
+ Iterator<CarbonDimension> iterator = filterComplexDimensions.iterator();
+ while (iterator.hasNext()) {
+ getChildDimensionDictionaryDetail(iterator.next(), dictionaryDimensionFromQuery);
+ }
+ // converting to list as api exposed needed list which i think
+ // is not correct
+ List<String> dictionaryColumnIdList =
+ new ArrayList<String>(dictionaryDimensionFromQuery.size());
+ dictionaryColumnIdList.addAll(dictionaryDimensionFromQuery);
+ return getDictionaryMap(dictionaryColumnIdList, absoluteTableIdentifier);
+ }
+
+ /**
+ * Below method will be used to fill the children dimension column id
+ *
+ * @param queryDimensions query dimension
+ * @param dictionaryDimensionFromQuery dictionary dimension for query
+ */
+ private static void getChildDimensionDictionaryDetail(CarbonDimension queryDimensions,
+ Set<String> dictionaryDimensionFromQuery) {
+ for (int j = 0; j < queryDimensions.numberOfChild(); j++) {
+ List<Encoding> encodingList = queryDimensions.getListOfChildDimensions().get(j).getEncoder();
+ if (queryDimensions.getListOfChildDimensions().get(j).numberOfChild() > 0) {
+ getChildDimensionDictionaryDetail(queryDimensions.getListOfChildDimensions().get(j),
+ dictionaryDimensionFromQuery);
+ } else if (!CarbonUtil.hasEncoding(encodingList, Encoding.DIRECT_DICTIONARY)) {
+ dictionaryDimensionFromQuery
+ .add(queryDimensions.getListOfChildDimensions().get(j).getColumnId());
+ }
+ }
+ }
+
+ /**
+ * Below method will be used to get the column id to its dictionary mapping
+ *
+ * @param dictionaryColumnIdList dictionary column list
+ * @param absoluteTableIdentifier absolute table identifier
+ * @return dictionary mapping
+ * @throws IOException
+ */
+ private static Map<String, Dictionary> getDictionaryMap(List<String> dictionaryColumnIdList,
+ AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException {
+ // this for dictionary unique identifier
+ List<DictionaryColumnUniqueIdentifier> dictionaryColumnUniqueIdentifiers =
+ getDictionaryColumnUniqueIdentifierList(dictionaryColumnIdList,
+ absoluteTableIdentifier.getCarbonTableIdentifier());
+ CacheProvider cacheProvider = CacheProvider.getInstance();
+ Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache = cacheProvider
+ .createCache(CacheType.FORWARD_DICTIONARY, absoluteTableIdentifier.getStorePath());
+
+ List<Dictionary> columnDictionaryList =
+ forwardDictionaryCache.getAll(dictionaryColumnUniqueIdentifiers);
+ Map<String, Dictionary> columnDictionaryMap = new HashMap<>(columnDictionaryList.size());
+ for (int i = 0; i < dictionaryColumnUniqueIdentifiers.size(); i++) {
+ // TODO: null check for column dictionary, if cache size is less it
+ // might return null here, in that case throw exception
+ columnDictionaryMap.put(dictionaryColumnIdList.get(i), columnDictionaryList.get(i));
+ }
+ return columnDictionaryMap;
+ }
+
+ /**
+ * Below method will be used to get the dictionary column unique identifier
+ *
+ * @param dictionaryColumnIdList dictionary
+ * @param carbonTableIdentifier
+ * @return
+ */
+ private static List<DictionaryColumnUniqueIdentifier> getDictionaryColumnUniqueIdentifierList(
+ List<String> dictionaryColumnIdList, CarbonTableIdentifier carbonTableIdentifier) {
+ CarbonTable carbonTable =
+ CarbonMetadata.getInstance().getCarbonTable(carbonTableIdentifier.getTableUniqueName());
+ List<DictionaryColumnUniqueIdentifier> dictionaryColumnUniqueIdentifiers =
+ new ArrayList<>(dictionaryColumnIdList.size());
+ for (String columnId : dictionaryColumnIdList) {
+ CarbonDimension dimension = CarbonMetadata.getInstance()
+ .getCarbonDimensionBasedOnColIdentifier(carbonTable, columnId);
+ if (dimension != null) {
+ dictionaryColumnUniqueIdentifiers.add(
+ new DictionaryColumnUniqueIdentifier(
+ carbonTableIdentifier,
+ dimension.getColumnIdentifier(),
+ dimension.getDataType()
+ )
+ );
+ }
+ }
+ return dictionaryColumnUniqueIdentifiers;
+ }
+
+ /**
+ * Below method will used to get the method will be used to get the measure
+ * block indexes to be read from the file
+ *
+ * @param queryMeasures query measure
+ * @param expressionMeasure measure present in the expression
+ * @param ordinalToBlockIndexMapping measure ordinal to block mapping
+ * @return block indexes
+ */
+ public static int[] getMeasureBlockIndexes(List<QueryMeasure> queryMeasures,
+ List<CarbonMeasure> expressionMeasure, Map<Integer, Integer> ordinalToBlockIndexMapping,
+ Set<CarbonMeasure> filterMeasures) {
+ Set<Integer> measureBlockIndex = new HashSet<Integer>();
+ Set<Integer> filterMeasureOrdinal = getFilterMeasureOrdinal(filterMeasures);
+ for (int i = 0; i < queryMeasures.size(); i++) {
+ if (!filterMeasureOrdinal.contains(queryMeasures.get(i).getMeasure().getOrdinal())) {
+ measureBlockIndex
+ .add(ordinalToBlockIndexMapping.get(queryMeasures.get(i).getMeasure().getOrdinal()));
+ }
+ }
+ for (int i = 0; i < expressionMeasure.size(); i++) {
+ measureBlockIndex.add(ordinalToBlockIndexMapping.get(expressionMeasure.get(i).getOrdinal()));
+ }
+ int[] measureIndexes =
+ ArrayUtils.toPrimitive(measureBlockIndex.toArray(new Integer[measureBlockIndex.size()]));
+ Arrays.sort(measureIndexes);
+ return measureIndexes;
+ }
+
+ /**
+ * Below method will be used to get mapping whether dimension is present in
+ * order by or not
+ *
+ * @param sortedDimensions sort dimension present in order by query
+ * @param queryDimensions query dimension
+ * @return sort dimension indexes
+ */
+ public static byte[] getSortDimensionIndexes(List<QueryDimension> sortedDimensions,
+ List<QueryDimension> queryDimensions) {
+ byte[] sortedDims = new byte[queryDimensions.size()];
+ int indexOf = 0;
+ for (int i = 0; i < sortedDims.length; i++) {
+ indexOf = sortedDimensions.indexOf(queryDimensions.get(i));
+ if (indexOf > -1) {
+ sortedDims[i] = 1;
+ }
+ }
+ return sortedDims;
+ }
+
+ /**
+ * Below method will be used to get the mapping of block index and its
+ * restructuring info
+ *
+ * @param queryDimensions query dimension from query model
+ * @param segmentProperties segment properties
+ * @return map of block index to its restructuring info
+ * @throws KeyGenException if problem while key generation
+ */
+ public static Map<Integer, KeyStructureInfo> getColumnGroupKeyStructureInfo(
+ List<QueryDimension> queryDimensions, SegmentProperties segmentProperties)
+ throws KeyGenException {
+ Map<Integer, KeyStructureInfo> rowGroupToItsRSInfo = new HashMap<Integer, KeyStructureInfo>();
+ // get column group id and its ordinal mapping of column group
+ Map<Integer, List<Integer>> columnGroupAndItsOrdinalMappingForQuery =
+ getColumnGroupAndItsOrdinalMapping(queryDimensions);
+ Map<Integer, KeyGenerator> columnGroupAndItsKeygenartor =
+ segmentProperties.getColumnGroupAndItsKeygenartor();
+
+ Iterator<Entry<Integer, List<Integer>>> iterator =
+ columnGroupAndItsOrdinalMappingForQuery.entrySet().iterator();
+ KeyStructureInfo restructureInfos = null;
+ while (iterator.hasNext()) {
+ Entry<Integer, List<Integer>> next = iterator.next();
+ KeyGenerator keyGenerator = columnGroupAndItsKeygenartor.get(next.getKey());
+ restructureInfos = new KeyStructureInfo();
+ // sort the ordinal
+ List<Integer> ordinal = next.getValue();
+ List<Integer> mdKeyOrdinal = new ArrayList<Integer>();
+ //Un sorted
+ List<Integer> mdKeyOrdinalForQuery = new ArrayList<Integer>();
+ for (Integer ord : ordinal) {
+ mdKeyOrdinal.add(segmentProperties.getColumnGroupMdKeyOrdinal(next.getKey(), ord));
+ mdKeyOrdinalForQuery.add(segmentProperties.getColumnGroupMdKeyOrdinal(next.getKey(), ord));
+ }
+ Collections.sort(mdKeyOrdinal);
+ // get the masked byte range for column group
+ int[] maskByteRanges = getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
+ // max key for column group
+ byte[] maxKey = getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator);
+ restructureInfos.setKeyGenerator(keyGenerator);
+ restructureInfos.setMaskByteRanges(maskByteRanges);
+ restructureInfos.setMaxKey(maxKey);
+ restructureInfos.setMdkeyQueryDimensionOrdinal(ArrayUtils
+ .toPrimitive(mdKeyOrdinalForQuery.toArray(new Integer[mdKeyOrdinalForQuery.size()])));
+ rowGroupToItsRSInfo
+ .put(segmentProperties.getDimensionOrdinalToBlockMapping().get(ordinal.get(0)),
+ restructureInfos);
+ }
+ return rowGroupToItsRSInfo;
+ }
+
+ /**
+ * return true if given key is found in array
+ *
+ * @param data
+ * @param key
+ * @return
+ */
+ public static boolean searchInArray(int[] data, int key) {
+ for (int i = 0; i < data.length; i++) {
+ if (key == data[i]) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Below method will be used to create a mapping of column group columns
+ * this mapping will have column group id to all the dimension ordinal
+ * present in the column group This mapping will be used during query
+ * execution, to create a mask key for the column group dimension which will
+ * be used in aggregation and filter query as column group dimension will be
+ * stored in bit level
+ */
+ private static Map<Integer, List<Integer>> getColumnGroupAndItsOrdinalMapping(
+ List<QueryDimension> origdimensions) {
+
+ List<QueryDimension> dimensions = new ArrayList<QueryDimension>(origdimensions.size());
+ dimensions.addAll(origdimensions);
+ /**
+ * sort based on column group id
+ */
+ Collections.sort(dimensions, new Comparator<QueryDimension>() {
+
+ @Override public int compare(QueryDimension o1, QueryDimension o2) {
+ return Integer
+ .compare(o1.getDimension().columnGroupId(), o2.getDimension().columnGroupId());
+ }
+ });
+ // list of row groups this will store all the row group column
+ Map<Integer, List<Integer>> columnGroupAndItsOrdinalsMapping =
+ new HashMap<Integer, List<Integer>>();
+ // to store a column group
+ List<Integer> currentColumnGroup = null;
+ // current index
+ int index = 0;
+ // previous column group to check all the column of row id has bee
+ // selected
+ int prvColumnGroupId = -1;
+ while (index < dimensions.size()) {
+ // if dimension group id is not zero and it is same as the previous
+ // column group id
+ // then we need to add ordinal of that column as it belongs to same
+ // column group
+ if (dimensions.get(index).getDimension().hasEncoding(Encoding.IMPLICIT)) {
+ index++;
+ continue;
+ } else if (!dimensions.get(index).getDimension().isColumnar()
+ && dimensions.get(index).getDimension().columnGroupId() == prvColumnGroupId
+ && null != currentColumnGroup) {
+ currentColumnGroup.add(dimensions.get(index).getDimension().getOrdinal());
+ }
+
+ // if dimension is not a columnar then it is column group column
+ else if (!dimensions.get(index).getDimension().isColumnar()) {
+ currentColumnGroup = new ArrayList<Integer>();
+ columnGroupAndItsOrdinalsMapping
+ .put(dimensions.get(index).getDimension().columnGroupId(), currentColumnGroup);
+ currentColumnGroup.add(dimensions.get(index).getDimension().getOrdinal());
+ }
+ // update the row id every time,this is required to group the
+ // columns
+ // of the same row group
+ prvColumnGroupId = dimensions.get(index).getDimension().columnGroupId();
+ index++;
+ }
+ return columnGroupAndItsOrdinalsMapping;
+ }
+
+ /**
+ * Below method will be used to get masked byte
+ *
+ * @param data actual data
+ * @param maxKey max key
+ * @param maskByteRanges mask byte range
+ * @param byteCount
+ * @return masked byte
+ */
+ public static byte[] getMaskedKey(byte[] data, byte[] maxKey, int[] maskByteRanges,
+ int byteCount) {
+ byte[] maskedKey = new byte[byteCount];
+ int counter = 0;
+ int byteRange = 0;
+ for (int i = 0; i < byteCount; i++) {
+ byteRange = maskByteRanges[i];
+ if (byteRange != -1) {
+ maskedKey[counter++] = (byte) (data[byteRange] & maxKey[byteRange]);
+ }
+ }
+ return maskedKey;
+ }
+
+ /**
+ * Below method will be used to fill block indexes of the query dimension
+ * which will be used in creating a output row Here is method we are passing
+ * two list which store the indexes one for dictionary column other for not
+ * dictionary column. This is done for specific purpose so that in one
+ * iteration we will be able to fill both type dimension block indexes
+ *
+ * @param queryDimensions dimension present in the query
+ * @param columnOrdinalToBlockIndexMapping column ordinal to block index mapping
+ * @param dictionaryDimensionBlockIndex list to store dictionary column block indexes
+ * @param noDictionaryDimensionBlockIndex list to store no dictionary block indexes
+ */
+ public static void fillQueryDimensionsBlockIndexes(List<QueryDimension> queryDimensions,
+ Map<Integer, Integer> columnOrdinalToBlockIndexMapping,
+ Set<Integer> dictionaryDimensionBlockIndex, List<Integer> noDictionaryDimensionBlockIndex) {
+ for (QueryDimension queryDimension : queryDimensions) {
+ if (CarbonUtil.hasEncoding(queryDimension.getDimension().getEncoder(), Encoding.DICTIONARY)
+ && queryDimension.getDimension().numberOfChild() == 0) {
+ dictionaryDimensionBlockIndex
+ .add(columnOrdinalToBlockIndexMapping.get(queryDimension.getDimension().getOrdinal()));
+ } else if (
+ !CarbonUtil.hasEncoding(queryDimension.getDimension().getEncoder(), Encoding.IMPLICIT)
+ && queryDimension.getDimension().numberOfChild() == 0) {
+ noDictionaryDimensionBlockIndex
+ .add(columnOrdinalToBlockIndexMapping.get(queryDimension.getDimension().getOrdinal()));
+ }
+ }
+ }
+
+ /**
+ * Below method will be used to resolve the query model
+ * resolve will be setting the actual dimension and measure object
+ * as from driver only column name will be passes to avoid the heavy object
+ * serialization
+ *
+ * @param queryModel query model
+ */
+ public static void resolveQueryModel(QueryModel queryModel) {
+ CarbonMetadata.getInstance().addCarbonTable(queryModel.getTable());
+ // TODO need to load the table from table identifier
+ CarbonTable carbonTable = queryModel.getTable();
+ String tableName =
+ queryModel.getAbsoluteTableIdentifier().getCarbonTableIdentifier().getTableName();
+ // resolve query dimension
+ for (QueryDimension queryDimension : queryModel.getQueryDimension()) {
+ queryDimension
+ .setDimension(carbonTable.getDimensionByName(tableName, queryDimension.getColumnName()));
+ }
+ // resolve query measure
+ for (QueryMeasure queryMeasure : queryModel.getQueryMeasures()) {
+ // in case of count start column name will be count * so
+ // first need to check any measure is present or not and as if measure
+ // if measure is present and if first measure is not a default
+ // measure than add measure otherwise
+ // than add first dimension as a measure
+ //as currently if measure is not present then
+ //we are adding default measure so first condition will
+ //never come false but if in future we can remove so not removing first if check
+ if (queryMeasure.getColumnName().equals("count(*)")) {
+ if (carbonTable.getMeasureByTableName(tableName).size() > 0 && !carbonTable
+ .getMeasureByTableName(tableName).get(0).getColName()
+ .equals(CarbonCommonConstants.DEFAULT_INVISIBLE_DUMMY_MEASURE)) {
+ queryMeasure.setMeasure(carbonTable.getMeasureByTableName(tableName).get(0));
+ } else {
+ CarbonMeasure dummyMeasure = new CarbonMeasure(
+ carbonTable.getDimensionByTableName(tableName).get(0).getColumnSchema(), 0);
+ queryMeasure.setMeasure(dummyMeasure);
+ }
+ } else {
+ queryMeasure
+ .setMeasure(carbonTable.getMeasureByName(tableName, queryMeasure.getColumnName()));
+ }
+ }
+ }
+
+ /**
+ * below method will be used to get the actual type aggregator
+ *
+ * @param aggType
+ * @return index in aggrgetor
+ */
+ public static int[] getActualTypeIndex(List<String> aggType) {
+ List<Integer> indexList = new ArrayList<Integer>();
+ for (int i = 0; i < aggType.size(); i++) {
+ if (!CarbonCommonConstants.SUM.equals(aggType.get(i)) && !CarbonCommonConstants.AVERAGE
+ .equals(aggType.get(i))) {
+ indexList.add(i);
+ }
+ }
+ return ArrayUtils.toPrimitive(indexList.toArray(new Integer[indexList.size()]));
+ }
+
+ /**
+ * Below method will be used to get the key structure for the column group
+ *
+ * @param segmentProperties segment properties
+ * @param dimColumnEvaluatorInfo dimension evaluator info
+ * @return key structure info for column group dimension
+ * @throws KeyGenException
+ */
+ public static KeyStructureInfo getKeyStructureInfo(SegmentProperties segmentProperties,
+ DimColumnResolvedFilterInfo dimColumnEvaluatorInfo) throws KeyGenException {
+ int colGrpId = getColumnGroupId(segmentProperties, dimColumnEvaluatorInfo.getColumnIndex());
+ KeyGenerator keyGenerator = segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
+ List<Integer> mdKeyOrdinal = new ArrayList<Integer>();
+
+ mdKeyOrdinal.add(segmentProperties
+ .getColumnGroupMdKeyOrdinal(colGrpId, dimColumnEvaluatorInfo.getColumnIndex()));
+ int[] maskByteRanges = QueryUtil.getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
+ byte[] maxKey = QueryUtil.getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator);
+ KeyStructureInfo restructureInfos = new KeyStructureInfo();
+ restructureInfos.setKeyGenerator(keyGenerator);
+ restructureInfos.setMaskByteRanges(maskByteRanges);
+ restructureInfos.setMaxKey(maxKey);
+ return restructureInfos;
+ }
+
+ /**
+ * Below method will be used to get the column group id based on the ordinal
+ *
+ * @param segmentProperties segment properties
+ * @param ordinal ordinal to be searched
+ * @return column group id
+ */
+ public static int getColumnGroupId(SegmentProperties segmentProperties, int ordinal) {
+ int[][] columnGroups = segmentProperties.getColumnGroups();
+ int colGrpId = -1;
+ for (int i = 0; i < columnGroups.length; i++) {
+ if (columnGroups[i].length > 1) {
+ colGrpId++;
+ if (QueryUtil.searchInArray(columnGroups[i], ordinal)) {
+ break;
+ }
+ }
+ }
+ return colGrpId;
+ }
+
+ /**
+ * Below method will be used to get the map of for complex dimension and its type
+ * which will be used to during query execution to
+ *
+ * @param queryDimensions complex dimension in query
+ * @param dimensionToBlockIndexMap dimension to block index in file map
+ * @return complex dimension and query type
+ */
+ public static Map<Integer, GenericQueryType> getComplexDimensionsMap(
+ List<QueryDimension> queryDimensions, Map<Integer, Integer> dimensionToBlockIndexMap,
+ int[] eachComplexColumnValueSize, Map<String, Dictionary> columnIdToDictionaryMap,
+ Set<CarbonDimension> filterDimensions) {
+ Map<Integer, GenericQueryType> complexTypeMap = new HashMap<Integer, GenericQueryType>();
+ for (QueryDimension dimension : queryDimensions) {
+ CarbonDimension actualDimension = dimension.getDimension();
+ if (actualDimension.getNumberOfChild() == 0) {
+ continue;
+ }
+ fillParentDetails(dimensionToBlockIndexMap, actualDimension, complexTypeMap,
+ eachComplexColumnValueSize, columnIdToDictionaryMap);
+ }
+ if (null != filterDimensions) {
+ for (CarbonDimension filterDimension : filterDimensions) {
+ // do not fill nay details for implicit dimension type
+ if (filterDimension.hasEncoding(Encoding.IMPLICIT)) {
+ continue;
+ }
+ fillParentDetails(dimensionToBlockIndexMap, filterDimension, complexTypeMap,
+ eachComplexColumnValueSize, columnIdToDictionaryMap);
+ }
+ }
+ return complexTypeMap;
+ }
+
+ private static void fillParentDetails(Map<Integer, Integer> dimensionToBlockIndexMap,
+ CarbonDimension dimension, Map<Integer, GenericQueryType> complexTypeMap,
+ int[] eachComplexColumnValueSize, Map<String, Dictionary> columnIdToDictionaryMap) {
+ int parentBlockIndex = dimensionToBlockIndexMap.get(dimension.getOrdinal());
+ GenericQueryType parentQueryType = dimension.getDataType().equals(DataType.ARRAY) ?
+ new ArrayQueryType(dimension.getColName(), dimension.getColName(), parentBlockIndex) :
+ new StructQueryType(dimension.getColName(), dimension.getColName(),
+ dimensionToBlockIndexMap.get(dimension.getOrdinal()));
+ complexTypeMap.put(dimension.getOrdinal(), parentQueryType);
+ parentBlockIndex =
+ fillChildrenDetails(eachComplexColumnValueSize, columnIdToDictionaryMap, parentBlockIndex,
+ dimension, parentQueryType);
+ }
+
+ private static int fillChildrenDetails(int[] eachComplexColumnValueSize,
+ Map<String, Dictionary> columnIdToDictionaryMap, int parentBlockIndex,
+ CarbonDimension dimension, GenericQueryType parentQueryType) {
+ for (int i = 0; i < dimension.getNumberOfChild(); i++) {
+ switch (dimension.getListOfChildDimensions().get(i).getDataType()) {
+ case ARRAY:
+ parentQueryType.addChildren(
+ new ArrayQueryType(dimension.getListOfChildDimensions().get(i).getColName(),
+ dimension.getColName(), ++parentBlockIndex));
+ break;
+ case STRUCT:
+ parentQueryType.addChildren(
+ new StructQueryType(dimension.getListOfChildDimensions().get(i).getColName(),
+ dimension.getColName(), ++parentBlockIndex));
+ break;
+ default:
+ boolean isDirectDictionary = CarbonUtil
+ .hasEncoding(dimension.getListOfChildDimensions().get(i).getEncoder(),
+ Encoding.DIRECT_DICTIONARY);
+ parentQueryType.addChildren(
+ new PrimitiveQueryType(dimension.getListOfChildDimensions().get(i).getColName(),
+ dimension.getColName(), ++parentBlockIndex,
+ dimension.getListOfChildDimensions().get(i).getDataType(),
+ eachComplexColumnValueSize[dimension.getListOfChildDimensions().get(i)
+ .getComplexTypeOrdinal()], columnIdToDictionaryMap
+ .get(dimension.getListOfChildDimensions().get(i).getColumnId()),
+ isDirectDictionary));
+ }
+ if (dimension.getListOfChildDimensions().get(i).getNumberOfChild() > 0) {
+ parentBlockIndex = fillChildrenDetails(eachComplexColumnValueSize, columnIdToDictionaryMap,
+ parentBlockIndex, dimension.getListOfChildDimensions().get(i), parentQueryType);
+ }
+ }
+ return parentBlockIndex;
+ }
+
+ public static void getAllFilterDimensions(FilterResolverIntf filterResolverTree,
+ Set<CarbonDimension> filterDimensions, Set<CarbonMeasure> filterMeasure) {
+ if (null == filterResolverTree) {
+ return;
+ }
+ List<ColumnExpression> dimensionResolvedInfos = new ArrayList<ColumnExpression>();
+ Expression filterExpression = filterResolverTree.getFilterExpression();
+ addColumnDimensions(filterExpression, filterDimensions, filterMeasure);
+ for (ColumnExpression info : dimensionResolvedInfos) {
+ if (info.isDimension() && info.getDimension().getNumberOfChild() > 0) {
+ filterDimensions.add(info.getDimension());
+ }
+ }
+ }
+
+ /**
+ * This method will check if a given expression contains a column expression
+ * recursively and add the dimension instance to the set which holds the dimension
+ * instances of the complex filter expressions.
+ */
+ private static void addColumnDimensions(Expression expression,
+ Set<CarbonDimension> filterDimensions, Set<CarbonMeasure> filterMeasure) {
+ if (null != expression && expression instanceof ColumnExpression) {
+ if (((ColumnExpression) expression).isDimension()) {
+ filterDimensions.add(((ColumnExpression) expression).getDimension());
+ } else {
+ filterMeasure.add((CarbonMeasure) ((ColumnExpression) expression).getCarbonColumn());
+ }
+ return;
+ } else if (null != expression) {
+ for (Expression child : expression.getChildren()) {
+ addColumnDimensions(child, filterDimensions, filterMeasure);
+ }
+ }
+ }
+
+ private static Set<Integer> getFilterMeasureOrdinal(Set<CarbonMeasure> filterMeasures) {
+ Set<Integer> filterMeasuresOrdinal = new HashSet<>();
+ for (CarbonMeasure filterMeasure : filterMeasures) {
+ filterMeasuresOrdinal.add(filterMeasure.getOrdinal());
+ }
+ return filterMeasuresOrdinal;
+ }
+
+ private static Set<Integer> getFilterDimensionOrdinal(Set<CarbonDimension> filterDimensions) {
+ Set<Integer> filterDimensionsOrdinal = new HashSet<>();
+ for (CarbonDimension filterDimension : filterDimensions) {
+ filterDimensionsOrdinal.add(filterDimension.getOrdinal());
+ getChildDimensionOrdinal(filterDimension, filterDimensionsOrdinal);
+ }
+ return filterDimensionsOrdinal;
+ }
+
+ /**
+ * Below method will be used to fill the children dimension column id
+ */
+ private static void getChildDimensionOrdinal(CarbonDimension queryDimensions,
+ Set<Integer> filterDimensionsOrdinal) {
+ for (int j = 0; j < queryDimensions.numberOfChild(); j++) {
+ List<Encoding> encodingList = queryDimensions.getListOfChildDimensions().get(j).getEncoder();
+ if (queryDimensions.getListOfChildDimensions().get(j).numberOfChild() > 0) {
+ getChildDimensionOrdinal(queryDimensions.getListOfChildDimensions().get(j),
+ filterDimensionsOrdinal);
+ } else if (!CarbonUtil.hasEncoding(encodingList, Encoding.DIRECT_DICTIONARY)) {
+ filterDimensionsOrdinal.add(queryDimensions.getListOfChildDimensions().get(j).getOrdinal());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
new file mode 100644
index 0000000..67835b6
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.executor.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.core.metadata.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.scan.executor.infos.AggregatorInfo;
+import org.apache.carbondata.core.scan.model.QueryDimension;
+import org.apache.carbondata.core.scan.model.QueryMeasure;
+
+/**
+ * Utility class for restructuring
+ */
+public class RestructureUtil {
+
+ /**
+ * Below method will be used to get the updated query dimension updation
+ * means, after restructuring some dimension will be not present in older
+ * table blocks in that case we need to select only those dimension out of
+ * query dimension which is present in the current table block
+ *
+ * @param queryDimensions
+ * @param tableBlockDimensions
+ * @return list of query dimension which is present in the table block
+ */
+ public static List<QueryDimension> getUpdatedQueryDimension(List<QueryDimension> queryDimensions,
+ List<CarbonDimension> tableBlockDimensions, List<CarbonDimension> tableComplexDimension) {
+ List<QueryDimension> presentDimension =
+ new ArrayList<QueryDimension>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ // selecting only those dimension which is present in the query
+ for (QueryDimension queryDimension : queryDimensions) {
+ if (queryDimension.getDimension().hasEncoding(Encoding.IMPLICIT)) {
+ presentDimension.add(queryDimension);
+ } else {
+ for (CarbonDimension tableDimension : tableBlockDimensions) {
+ if (tableDimension.equals(queryDimension.getDimension())) {
+ presentDimension.add(queryDimension);
+ }
+ }
+ }
+ }
+ for (QueryDimension queryDimimension : queryDimensions) {
+ for (CarbonDimension tableDimension : tableComplexDimension) {
+ if (tableDimension.equals(queryDimimension.getDimension())) {
+ presentDimension.add(queryDimimension);
+ }
+ }
+ }
+ return presentDimension;
+ }
+
+ /**
+ * Below method is to add dimension children for complex type dimension as
+ * internally we are creating dimension column for each each complex
+ * dimension so when complex query dimension request will come in the query,
+ * we need to add its children as it is hidden from the user For example if
+ * complex dimension is of Array of String[2] so we are storing 3 dimension
+ * and when user will query for complex type i.e. array type we need to add
+ * its children and then we will read respective block and create a tuple
+ * based on all three dimension
+ *
+ * @param queryDimensions current query dimensions
+ * @param tableBlockDimensions dimensions which is present in the table block
+ * @return updated dimension(after adding complex type children)
+ */
+ public static List<CarbonDimension> addChildrenForComplexTypeDimension(
+ List<CarbonDimension> queryDimensions, List<CarbonDimension> tableBlockDimensions) {
+ List<CarbonDimension> updatedQueryDimension = new ArrayList<CarbonDimension>();
+ int numberOfChildren = 0;
+ for (CarbonDimension queryDimension : queryDimensions) {
+ // if number of child is zero, then it is not a complex dimension
+ // so directly add it query dimension
+ if (queryDimension.numberOfChild() == 0) {
+ updatedQueryDimension.add(queryDimension);
+ }
+ // if number of child is more than 1 then add all its children
+ numberOfChildren = queryDimension.getOrdinal() + queryDimension.numberOfChild();
+ for (int j = queryDimension.getOrdinal(); j < numberOfChildren; j++) {
+ updatedQueryDimension.add(tableBlockDimensions.get(j));
+ }
+ }
+ return updatedQueryDimension;
+ }
+
+ /**
+ * Below method will be used to get the aggregator info object
+ * in this method some of the properties which will be extracted
+ * from query measure and current block measures will be set
+ *
+ * @param queryMeasures measures present in query
+ * @param currentBlockMeasures current block measures
+ * @return aggregator info
+ */
+ public static AggregatorInfo getAggregatorInfos(List<QueryMeasure> queryMeasures,
+ List<CarbonMeasure> currentBlockMeasures) {
+ AggregatorInfo aggregatorInfos = new AggregatorInfo();
+ int numberOfMeasureInQuery = queryMeasures.size();
+ int[] measureOrdinals = new int[numberOfMeasureInQuery];
+ Object[] defaultValues = new Object[numberOfMeasureInQuery];
+ boolean[] measureExistsInCurrentBlock = new boolean[numberOfMeasureInQuery];
+ int index = 0;
+ for (QueryMeasure queryMeasure : queryMeasures) {
+ measureOrdinals[index] = queryMeasure.getMeasure().getOrdinal();
+ // if query measure exists in current dimension measures
+ // then setting measure exists is true
+ // otherwise adding a default value of a measure
+ if (currentBlockMeasures.contains(queryMeasure.getMeasure())) {
+ measureExistsInCurrentBlock[index] = true;
+ } else {
+ defaultValues[index] = queryMeasure.getMeasure().getDefaultValue();
+ }
+ index++;
+ }
+ aggregatorInfos.setDefaultValues(defaultValues);
+ aggregatorInfos.setMeasureOrdinals(measureOrdinals);
+ aggregatorInfos.setMeasureExists(measureExistsInCurrentBlock);
+ return aggregatorInfos;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/expression/BinaryExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/BinaryExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/BinaryExpression.java
new file mode 100644
index 0000000..8885841
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/BinaryExpression.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression;
+
+public abstract class BinaryExpression extends Expression {
+
+ private static final long serialVersionUID = 1L;
+ protected Expression left;
+ protected Expression right;
+
+ public BinaryExpression(Expression left, Expression right) {
+ this.left = left;
+ this.right = right;
+ children.add(left);
+ children.add(right);
+ }
+
+ public Expression getLeft() {
+ return left;
+ }
+
+ public Expression getRight() {
+ return right;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java
new file mode 100644
index 0000000..4637abb
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/ColumnExpression.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+
+public class ColumnExpression extends LeafExpression {
+
+ private static final long serialVersionUID = 1L;
+
+ private String columnName;
+
+ private boolean isDimension;
+
+ private int colIndex = -1;
+
+ private DataType dataType;
+
+ private CarbonDimension dimension;
+
+ private CarbonColumn carbonColumn;
+
+ public ColumnExpression(String columnName, DataType dataType) {
+ this.columnName = columnName;
+ this.dataType = dataType;
+
+ }
+
+ public CarbonDimension getDimension() {
+ return dimension;
+ }
+
+ public void setDimension(CarbonDimension dimension) {
+ this.dimension = dimension;
+ }
+
+ public String getColumnName() {
+ return columnName;
+ }
+
+ public void setColumnName(String columnName) {
+ this.columnName = columnName;
+ }
+
+ public boolean isDimension() {
+ return isDimension;
+ }
+
+ public void setDimension(boolean isDimension) {
+ this.isDimension = isDimension;
+ }
+
+ public int getColIndex() {
+ return colIndex;
+ }
+
+ public void setColIndex(int colIndex) {
+ this.colIndex = colIndex;
+ }
+
+ public DataType getDataType() {
+ return dataType;
+ }
+
+ public void setDataType(DataType dataType) {
+ this.dataType = dataType;
+ }
+
+ @Override public ExpressionResult evaluate(RowIntf value) {
+ ExpressionResult expressionResult =
+ new ExpressionResult(dataType, (null == value ? null : value.getVal(colIndex)));
+ return expressionResult;
+ }
+
+ @Override public ExpressionType getFilterExpressionType() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override public String getString() {
+ // TODO Auto-generated method stub
+ return "ColumnExpression(" + columnName + ')';
+ }
+
+ public CarbonColumn getCarbonColumn() {
+ return carbonColumn;
+ }
+
+ public void setCarbonColumn(CarbonColumn carbonColumn) {
+ this.carbonColumn = carbonColumn;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/expression/Expression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/Expression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/Expression.java
new file mode 100644
index 0000000..a816643
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/Expression.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+
+public abstract class Expression implements Serializable {
+
+ private static final long serialVersionUID = -7568676723039530713L;
+ protected List<Expression> children =
+ new ArrayList<Expression>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+
+ public abstract ExpressionResult evaluate(RowIntf value)
+ throws FilterUnsupportedException, FilterIllegalMemberException;
+
+ public abstract ExpressionType getFilterExpressionType();
+
+ public List<Expression> getChildren() {
+ return children;
+ }
+
+ public abstract String getString();
+
+}
[08/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryTypeTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryTypeTest.java b/core/src/test/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryTypeTest.java
new file mode 100644
index 0000000..b8f484f
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryTypeTest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.complextypes;
+
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.core.cache.dictionary.ColumnDictionaryInfo;
+import org.apache.carbondata.core.cache.dictionary.Dictionary;
+import org.apache.carbondata.core.cache.dictionary.ForwardDictionary;
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
+import org.apache.carbondata.core.keygenerator.mdkey.Bits;
+import org.apache.carbondata.core.util.DataTypeUtil;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class PrimitiveQueryTypeTest {
+ private static PrimitiveQueryType primitiveQueryType, primitiveQueryTypeForInt,
+ primitiveQueryTypeForLong, primitiveQueryTypeForDouble, primitiveQueryTypeForBoolean,
+ primitiveQueryTypeForTimeStamp, primitiveQueryTypeForTimeStampForIsDictionaryFalse;
+ private static Dictionary dictionary;
+ private boolean isDirectDictionary = false;
+
+ @BeforeClass public static void setUp() {
+ String name = "test";
+ String parentName = "testParent";
+ int blockIndex = 1;
+ int keySize = 1;
+ boolean isDirectDictionary = true;
+ primitiveQueryType =
+ new PrimitiveQueryType(name, parentName, blockIndex, DataType.STRING, keySize, dictionary,
+ isDirectDictionary);
+ primitiveQueryTypeForInt =
+ new PrimitiveQueryType(name, parentName, blockIndex, DataType.INT, keySize, dictionary,
+ isDirectDictionary);
+ primitiveQueryTypeForDouble =
+ new PrimitiveQueryType(name, parentName, blockIndex, DataType.DOUBLE, keySize, dictionary,
+ isDirectDictionary);
+ primitiveQueryTypeForLong =
+ new PrimitiveQueryType(name, parentName, blockIndex, DataType.LONG, keySize, dictionary,
+ isDirectDictionary);
+ primitiveQueryTypeForBoolean =
+ new PrimitiveQueryType(name, parentName, blockIndex, DataType.BOOLEAN, keySize, dictionary,
+ isDirectDictionary);
+ primitiveQueryTypeForTimeStamp =
+ new PrimitiveQueryType(name, parentName, blockIndex, DataType.TIMESTAMP, keySize,
+ dictionary, isDirectDictionary);
+ ColumnDictionaryInfo columnDictionaryInfo = new ColumnDictionaryInfo(DataType.STRING);
+ ForwardDictionary forwardDictionary = new ForwardDictionary(columnDictionaryInfo);
+ primitiveQueryTypeForTimeStampForIsDictionaryFalse =
+ new PrimitiveQueryType(name, parentName, blockIndex, DataType.TIMESTAMP, keySize,
+ forwardDictionary, false);
+
+ }
+
+ @Test public void testGetDataBasedOnDataTypeFromSurrogates() {
+ ByteBuffer surrogateData = ByteBuffer.allocate(10);
+ surrogateData.put(3, (byte) 1);
+ new MockUp<Bits>() {
+ @Mock public long[] getKeyArray(byte[] key, int offset) {
+ return new long[] { 1313045L };
+ }
+ };
+ Object expectedValue = 1313043000000L;
+
+ Object actualValue =
+ primitiveQueryTypeForTimeStamp.getDataBasedOnDataTypeFromSurrogates(surrogateData);
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test public void testGetDataBasedOnDataTypeFromSurrogatesWhenIsDictionaryFalse() {
+ ByteBuffer surrogateData = ByteBuffer.allocate(10);
+ surrogateData.put(3, (byte) 1);
+ new MockUp<ForwardDictionary>() {
+ @Mock public String getDictionaryValueForKey(int surrogateKey) {
+ return "2015-10-20 12:30:01";
+ }
+ };
+
+ Object expectedValue = primitiveQueryTypeForTimeStampForIsDictionaryFalse
+ .getDataBasedOnDataTypeFromSurrogates(surrogateData);
+ Object actualValue = primitiveQueryTypeForTimeStampForIsDictionaryFalse
+ .getDataBasedOnDataTypeFromSurrogates(surrogateData);
+ assertEquals(expectedValue, actualValue);
+ }
+
+ public Object getDataBasedOnDataTypeFromSurrogates(ByteBuffer surrogateData) {
+ int keySize = 2;
+ byte[] data = new byte[keySize];
+ surrogateData.get(data);
+ Bits bit = new Bits(new int[] { keySize * 8 });
+ int surrgateValue = (int) bit.getKeyArray(data, 0)[0];
+ Object actualData = null;
+ if (isDirectDictionary) {
+ DirectDictionaryGenerator directDictionaryGenerator =
+ DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(DataType.TIMESTAMP);
+ actualData = directDictionaryGenerator.getValueFromSurrogate(surrgateValue);
+ } else {
+ String dictionaryValueForKey = dictionary.getDictionaryValueForKey(surrgateValue);
+ actualData = DataTypeUtil.getDataBasedOnDataType(dictionaryValueForKey, DataType.TIMESTAMP);
+ }
+ return actualData;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/scan/complextypes/StructQueryTypeTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/complextypes/StructQueryTypeTest.java b/core/src/test/java/org/apache/carbondata/core/scan/complextypes/StructQueryTypeTest.java
new file mode 100644
index 0000000..2a9d724
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/complextypes/StructQueryTypeTest.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.complextypes;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public class StructQueryTypeTest {
+ private static StructQueryType structQueryType;
+ private static ArrayQueryType arrayQueryType;
+
+ @BeforeClass public static void setUp() {
+ String name = "testName";
+ String parentName = "testName";
+ int blockIndex = 5;
+ structQueryType = new StructQueryType(name, parentName, blockIndex);
+ arrayQueryType = new ArrayQueryType(name, parentName, blockIndex);
+ }
+
+ @Test public void testGetDataBasedOnDataTypeFromSurrogates() {
+ ByteBuffer surrogateData = ByteBuffer.allocate(10);
+ surrogateData.put(3, (byte) 1);
+ structQueryType.addChildren(arrayQueryType);
+ List children = new ArrayList();
+ children.add(arrayQueryType);
+ assertNotNull(structQueryType.getDataBasedOnDataTypeFromSurrogates(surrogateData));
+ }
+
+ @Test public void testGetColsCount() {
+ structQueryType.setName("testName");
+ structQueryType.setParentname("testName");
+ structQueryType.addChildren(arrayQueryType);
+ new MockUp<ArrayQueryType>() {
+ @Mock int getColsCount() {
+ return 1;
+ }
+ };
+ int actualValue = structQueryType.getColsCount();
+ int expectedValue = 2;
+ assertEquals(expectedValue, actualValue);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
new file mode 100644
index 0000000..00f05f4
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/QueryUtilTest.java
@@ -0,0 +1,241 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.executor.util;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.block.SegmentPropertiesTestUtil;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.scan.model.QueryDimension;
+
+import junit.framework.TestCase;
+import mockit.Mock;
+import mockit.MockUp;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+
+public class QueryUtilTest extends TestCase {
+
+ private SegmentProperties segmentProperties;
+
+ @BeforeClass public void setUp() {
+ segmentProperties = SegmentPropertiesTestUtil.getSegmentProperties();
+ }
+
+ @Test public void testGetMaskedByteRangeGivingProperMaksedByteRange() {
+
+ QueryDimension dimension =
+ new QueryDimension(segmentProperties.getDimensions().get(0).getColName());
+ dimension.setDimension(segmentProperties.getDimensions().get(0));
+ int[] maskedByteRange = QueryUtil
+ .getMaskedByteRange(Arrays.asList(dimension), segmentProperties.getDimensionKeyGenerator());
+ int[] expectedMaskedByteRange = { 0 };
+ for (int i = 0; i < maskedByteRange.length; i++) {
+ assertEquals(expectedMaskedByteRange[i], maskedByteRange[i]);
+ }
+ }
+
+ @Test public void testGetMaskedByteRangeGivingProperMaksedByteRangeOnlyForDictionaryKey() {
+ List<QueryDimension> dimensions = new ArrayList<QueryDimension>();
+ for (int i = 0; i < 2; i++) {
+ QueryDimension dimension =
+ new QueryDimension(segmentProperties.getDimensions().get(i).getColName());
+ dimension.setDimension(segmentProperties.getDimensions().get(i));
+ dimensions.add(dimension);
+ }
+ int[] maskedByteRange =
+ QueryUtil.getMaskedByteRange(dimensions, segmentProperties.getDimensionKeyGenerator());
+ int[] expectedMaskedByteRange = { 0 };
+ for (int i = 0; i < maskedByteRange.length; i++) {
+ assertEquals(expectedMaskedByteRange[i], maskedByteRange[i]);
+ }
+ }
+
+ @Test public void testGetMaskedByteRangeBasedOrdinalGivingProperMaskedByte() {
+ List<Integer> dimensionOrdinal = new ArrayList<Integer>();
+ dimensionOrdinal.add(0);
+ int[] maskedByteRange = QueryUtil.getMaskedByteRangeBasedOrdinal(dimensionOrdinal,
+ segmentProperties.getDimensionKeyGenerator());
+ int[] expectedMaskedByteRange = { 0 };
+ for (int i = 0; i < maskedByteRange.length; i++) {
+ assertEquals(expectedMaskedByteRange[i], maskedByteRange[i]);
+ }
+ }
+
+ @Test public void testGetMaxKeyBasedOnDimensions() {
+ List<QueryDimension> dimensions = new ArrayList<QueryDimension>();
+ for (int i = 0; i < 2; i++) {
+ QueryDimension dimension =
+ new QueryDimension(segmentProperties.getDimensions().get(i).getColName());
+ dimension.setDimension(segmentProperties.getDimensions().get(i));
+ dimensions.add(dimension);
+ }
+ byte[] maxKeyBasedOnDimensions = null;
+ try {
+ maxKeyBasedOnDimensions = QueryUtil
+ .getMaxKeyBasedOnDimensions(dimensions, segmentProperties.getDimensionKeyGenerator());
+ } catch (KeyGenException e) {
+ assertTrue(false);
+ }
+ byte[] expectedMaxKeyBasedOnDimensions = { -1, 0, 0, 0, 0, 0 };
+ for (int i = 0; i < expectedMaxKeyBasedOnDimensions.length; i++) {
+ if (expectedMaxKeyBasedOnDimensions[i] != maxKeyBasedOnDimensions[i]) {
+ assertTrue(false);
+ }
+ }
+ long[] expectedKeyArray = { 255, 0, 0, 0, 0, 0 };
+ long[] keyArray =
+ segmentProperties.getDimensionKeyGenerator().getKeyArray(maxKeyBasedOnDimensions);
+ for (int i = 0; i < keyArray.length; i++) {
+ if (expectedKeyArray[i] != keyArray[i]) {
+ assertTrue(false);
+ }
+ }
+ }
+
+ @Test public void testGetMaksedByte() {
+ QueryDimension dimension =
+ new QueryDimension(segmentProperties.getDimensions().get(0).getColName());
+ dimension.setDimension(segmentProperties.getDimensions().get(0));
+ dimension.setDimension(segmentProperties.getDimensions().get(0));
+ int[] maskedByteRange = QueryUtil
+ .getMaskedByteRange(Arrays.asList(dimension), segmentProperties.getDimensionKeyGenerator());
+ int[] maskedByte = QueryUtil
+ .getMaskedByte(segmentProperties.getDimensionKeyGenerator().getDimCount(), maskedByteRange);
+ int[] expectedMaskedByte = { 0, -1, -1, -1, -1, -1 };
+
+ for (int i = 0; i < expectedMaskedByte.length; i++) {
+ if (expectedMaskedByte[i] != maskedByte[i]) {
+ assertTrue(false);
+ }
+ }
+ }
+
+ @Test public void testSearchInArrayWithSearchInputNotPresentInArray() {
+ int[] dummyArray = { 1, 2, 3, 4, 5 };
+ int searchInput = 6;
+ boolean result = QueryUtil.searchInArray(dummyArray, searchInput);
+ assert (!result);
+ }
+
+ @Test public void testSearchInArrayWithSearchInputPresentInArray() {
+ int[] dummyArray = { 1, 2, 3, 4, 5 };
+ int searchInput = 1;
+ boolean result = QueryUtil.searchInArray(dummyArray, searchInput);
+ assert (result);
+ }
+
+ @Test public void testGetColumnGroupIdWhenOrdinalValueNotPresentInArrayIndex() {
+ int ordinal = 0;
+ new MockUp<SegmentProperties>() {
+ @Mock public int[][] getColumnGroups() {
+ int columnGroups[][] = { { 1, 1 }, { 2, 2 }, { 3, 3 }, { 4, 4 }, { 5, 5 } };
+ return columnGroups;
+ }
+ };
+ int actualValue = QueryUtil.getColumnGroupId(segmentProperties, ordinal);
+ int expectedValue = 4; //expectedValue will always be arrayLength - 1
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test public void testGetColumnGroupIdWhenOrdinalValuePresentInArrayIndex() {
+ int ordinal = 1;
+ new MockUp<SegmentProperties>() {
+ @Mock public int[][] getColumnGroups() {
+ int columnGroups[][] = { { 1, 1 }, { 2, 2 }, { 3, 3 }, { 4, 4 }, { 5, 5 } };
+ return columnGroups;
+ }
+ };
+ int actualValue = QueryUtil.getColumnGroupId(segmentProperties, ordinal);
+ int expectedValue = 0;
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test public void testGetColumnGroupIdWhenColumnGroupsIndexValueLengthLessThanOne() {
+ int ordinal = 1;
+ new MockUp<SegmentProperties>() {
+ @Mock public int[][] getColumnGroups() {
+ int columnGroups[][] = { { 1 } };
+ return columnGroups;
+ }
+ };
+ int actualValue = QueryUtil.getColumnGroupId(segmentProperties, ordinal);
+ int expectedValue = -1;
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test public void testGetMaskedKey() {
+ byte[] data = { 1, 2, 3, 4, 5, 5 };
+ byte[] maxKey = { 15, 20, 25, 30, 35, 35 };
+ int[] maskByteRanges = { 1, 2, 3, 4, 5 };
+ int byteCount = 5;
+ byte[] actualValue = QueryUtil.getMaskedKey(data, maxKey, maskByteRanges, byteCount);
+ byte[] expectedValue = { 0, 1, 4, 1, 1 };
+ assertArrayEquals(expectedValue, actualValue);
+ }
+
+ @Test public void testGetMaxKeyBasedOnOrinal() throws Exception {
+ List<Integer> dummyList = new ArrayList<>();
+ dummyList.add(0, 1);
+ dummyList.add(1, 2);
+ byte[] actualValue =
+ QueryUtil.getMaxKeyBasedOnOrinal(dummyList, segmentProperties.getDimensionKeyGenerator());
+ byte[] expectedValue = { 0, -1, -1, 0, 0, 0 };
+ assertArrayEquals(expectedValue, actualValue);
+ }
+
+ @Test public void testGetSortDimensionIndexes() {
+ List<QueryDimension> sortedDimensions = new ArrayList<QueryDimension>();
+ for (int i = 0; i < 2; i++) {
+ QueryDimension dimension =
+ new QueryDimension(segmentProperties.getDimensions().get(i).getColName());
+ dimension.setDimension(segmentProperties.getDimensions().get(i));
+ sortedDimensions.add(dimension);
+ }
+ List<QueryDimension> queryDimensions = new ArrayList<QueryDimension>();
+ for (int i = 0; i < 2; i++) {
+ QueryDimension dimension =
+ new QueryDimension(segmentProperties.getDimensions().get(i).getColName());
+ dimension.setDimension(segmentProperties.getDimensions().get(i));
+ queryDimensions.add(dimension);
+ }
+ byte[] actualValue = QueryUtil.getSortDimensionIndexes(sortedDimensions, queryDimensions);
+ byte[] expectedValue = { 0, 0 };
+ assertArrayEquals(expectedValue, actualValue);
+ }
+
+ @Test public void testGetActualTypeIndex() {
+ List<String> dummyList = new ArrayList<>();
+ dummyList.add("test1");
+ dummyList.add("test2");
+ int[] actualValue = QueryUtil.getActualTypeIndex(dummyList);
+ int[] expectedValue = { 0, 1 };
+ assertArrayEquals(expectedValue, actualValue);
+ }
+
+ @AfterClass public void tearDown() {
+ segmentProperties = null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java
new file mode 100644
index 0000000..ab17b43
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/executor/util/RestructureUtilTest.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.executor.util;
+
+import org.apache.carbondata.core.metadata.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.scan.executor.infos.AggregatorInfo;
+import org.apache.carbondata.core.scan.model.QueryDimension;
+import org.apache.carbondata.core.scan.model.QueryMeasure;
+
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class RestructureUtilTest {
+
+ @Test public void testToGetUpdatedQueryDimension() {
+ List<Encoding> encodingList = new ArrayList<Encoding>();
+ encodingList.add(Encoding.DICTIONARY);
+ ColumnSchema columnSchema1 = new ColumnSchema();
+ columnSchema1.setColumnName("Id");
+ columnSchema1.setEncodingList(encodingList);
+ ColumnSchema columnSchema2 = new ColumnSchema();
+ columnSchema2.setColumnName("Name");
+ columnSchema2.setEncodingList(encodingList);
+ ColumnSchema columnSchema3 = new ColumnSchema();
+ columnSchema3.setColumnName("Age");
+ columnSchema3.setEncodingList(encodingList);
+ ColumnSchema columnSchema4 = new ColumnSchema();
+ columnSchema4.setColumnName("Salary");
+ columnSchema4.setEncodingList(encodingList);
+ ColumnSchema columnSchema5 = new ColumnSchema();
+ columnSchema5.setColumnName("Address");
+ columnSchema5.setEncodingList(encodingList);
+
+ CarbonDimension tableBlockDimension1 = new CarbonDimension(columnSchema1, 1, 1, 1, 1);
+ CarbonDimension tableBlockDimension2 = new CarbonDimension(columnSchema2, 5, 5, 5, 5);
+ List<CarbonDimension> tableBlockDimensions =
+ Arrays.asList(tableBlockDimension1, tableBlockDimension2);
+
+ CarbonDimension tableComplexDimension1 = new CarbonDimension(columnSchema3, 4, 4, 4, 4);
+ CarbonDimension tableComplexDimension2 = new CarbonDimension(columnSchema4, 2, 2, 2, 2);
+ List<CarbonDimension> tableComplexDimensions =
+ Arrays.asList(tableComplexDimension1, tableComplexDimension2);
+
+ QueryDimension queryDimension1 = new QueryDimension("Id");
+ queryDimension1.setDimension(tableBlockDimension1);
+ QueryDimension queryDimension2 = new QueryDimension("Name");
+ queryDimension2.setDimension(tableComplexDimension2);
+ QueryDimension queryDimension3 = new QueryDimension("Address");
+ queryDimension3.setDimension(new CarbonDimension(columnSchema5, 3, 3, 3, 3));
+
+ List<QueryDimension> queryDimensions =
+ Arrays.asList(queryDimension1, queryDimension2, queryDimension3);
+
+ List<QueryDimension> result = RestructureUtil
+ .getUpdatedQueryDimension(queryDimensions, tableBlockDimensions, tableComplexDimensions);
+
+ assertThat(result, is(equalTo(Arrays.asList(queryDimension1, queryDimension2))));
+ }
+
+ @Test public void testToGetAggregatorInfos() {
+ ColumnSchema columnSchema1 = new ColumnSchema();
+ columnSchema1.setColumnName("Id");
+ ColumnSchema columnSchema2 = new ColumnSchema();
+ columnSchema2.setColumnName("Name");
+ ColumnSchema columnSchema3 = new ColumnSchema();
+ columnSchema3.setColumnName("Age");
+
+ CarbonMeasure carbonMeasure1 = new CarbonMeasure(columnSchema1, 1);
+ CarbonMeasure carbonMeasure2 = new CarbonMeasure(columnSchema2, 2);
+ CarbonMeasure carbonMeasure3 = new CarbonMeasure(columnSchema3, 3);
+ carbonMeasure3.setDefaultValue("3".getBytes());
+ List<CarbonMeasure> currentBlockMeasures = Arrays.asList(carbonMeasure1, carbonMeasure2);
+
+ QueryMeasure queryMeasure1 = new QueryMeasure("Id");
+ queryMeasure1.setMeasure(carbonMeasure1);
+ QueryMeasure queryMeasure2 = new QueryMeasure("Name");
+ queryMeasure2.setMeasure(carbonMeasure2);
+ QueryMeasure queryMeasure3 = new QueryMeasure("Age");
+ queryMeasure3.setMeasure(carbonMeasure3);
+ List<QueryMeasure> queryMeasures = Arrays.asList(queryMeasure1, queryMeasure2, queryMeasure3);
+
+ AggregatorInfo aggregatorInfo =
+ RestructureUtil.getAggregatorInfos(queryMeasures, currentBlockMeasures);
+ boolean[] measuresExist = { true, true, false };
+ assertThat(aggregatorInfo.getMeasureExists(), is(equalTo(measuresExist)));
+ Object[] defaultValues = { null, null, "3".getBytes() };
+ assertThat(aggregatorInfo.getDefaultValues(), is(equalTo(defaultValues)));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/scan/expression/ColumnExpressionTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/ColumnExpressionTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/ColumnExpressionTest.java
new file mode 100644
index 0000000..3209991
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/ColumnExpressionTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.scan.filter.intf.RowImpl;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class ColumnExpressionTest {
+
+ private ColumnExpression columnExpression;
+
+ @Before public void setUp() {
+ String columnName = "IMEI";
+ DataType dataType = DataType.STRING;
+ columnExpression = new ColumnExpression(columnName, dataType);
+ }
+
+ @Test public void testEvaluate() {
+ RowImpl rowImpl = new RowImpl();
+ rowImpl.setValues(new Integer[] { 1 });
+ new MockUp<RowImpl>() {
+ @Mock public Object getVal(int index) {
+ return 1;
+ }
+ };
+ ExpressionResult expectedValue = new ExpressionResult(DataType.INT, 1);
+ assertEquals(expectedValue, columnExpression.evaluate(rowImpl));
+ }
+
+ @Test public void testEvaluateForNullValue() {
+ RowImpl rowImpl = new RowImpl();
+ rowImpl.setValues(null);
+ new MockUp<RowImpl>() {
+ @Mock public Object getVal(int index) {
+ return null;
+ }
+ };
+ ExpressionResult expectedValue = new ExpressionResult(null);
+ assertEquals(expectedValue, columnExpression.evaluate(rowImpl));
+ }
+
+ @Test public void testGetString() {
+ String actualValue = columnExpression.getString();
+ String expectedValue = "ColumnExpression(IMEI)";
+ assertEquals(expectedValue, actualValue);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/scan/expression/ExpressionResultTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/ExpressionResultTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/ExpressionResultTest.java
new file mode 100644
index 0000000..8f0fa02
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/ExpressionResultTest.java
@@ -0,0 +1,498 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.expression;
+
+import java.math.BigDecimal;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.IsEqual.equalTo;
+import static org.junit.Assert.assertEquals;
+import static org.pentaho.di.core.util.Assert.assertFalse;
+import static org.pentaho.di.core.util.Assert.assertNull;
+
+public class ExpressionResultTest {
+ private static final double DELTA = 1e-15;
+ private static ExpressionResult expressionResult;
+
+ @BeforeClass public static void setUp() {
+ expressionResult = new ExpressionResult(DataType.INT, null);
+ }
+
+ @Test public void testGetIntForNull() throws FilterIllegalMemberException {
+ assertNull(expressionResult.getInt());
+ }
+
+ @Test public void testGetIntForString() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForString = new ExpressionResult(DataType.STRING, "5");
+ int actualValue = expressionResultForString.getInt();
+ int expectedValue = 5;
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test(expected = FilterIllegalMemberException.class)
+ public void testGetIntForNumberFormatException() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForString = new ExpressionResult(DataType.STRING, "FOOBAR");
+ expressionResultForString.getInt();
+ }
+
+ @Test public void testGetIntForDouble() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForDouble = new ExpressionResult(DataType.DOUBLE, 5.0);
+ int actualValue = expressionResultForDouble.getInt();
+ int expectedValue = 5;
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test public void testGetIntForInt() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForInt = new ExpressionResult(DataType.INT, 5);
+ int actualValue = expressionResultForInt.getInt();
+ int expectedValue = 5;
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test(expected = FilterIllegalMemberException.class)
+ public void testGetIntForIllegalMemberException() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForInt = new ExpressionResult(DataType.FLOAT, 25.36);
+ expressionResultForInt.getInt();
+ }
+
+ @Test(expected = FilterIllegalMemberException.class) public void testGetIntForClassCastException()
+ throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForInt = new ExpressionResult(DataType.INT, "STRING");
+ expressionResultForInt.getInt();
+ }
+
+ @Test public void testGetShortForNull() throws FilterIllegalMemberException {
+ assertNull(expressionResult.getShort());
+ }
+
+ @Test public void testGetShortForString() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForString = new ExpressionResult(DataType.STRING, "5");
+ short actualValue = expressionResultForString.getShort();
+ short expectedValue = 5;
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test(expected = FilterIllegalMemberException.class)
+ public void testGetShortForNumberFormatException() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForString = new ExpressionResult(DataType.STRING, "FOOBAR");
+ expressionResultForString.getShort();
+ }
+
+ @Test public void testGetShortForDouble() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForDouble = new ExpressionResult(DataType.DOUBLE, 5.0);
+ short actualValue = expressionResultForDouble.getShort();
+ short expectedValue = 5;
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test public void testGetShortForInt() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForInt = new ExpressionResult(DataType.INT, 5);
+ short actualValue = expressionResultForInt.getShort();
+ short expectedValue = 5;
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test(expected = FilterIllegalMemberException.class)
+ public void testGetShortForIllegalMemberException() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForInt = new ExpressionResult(DataType.FLOAT, 25.36);
+ expressionResultForInt.getShort();
+ }
+
+ @Test(expected = FilterIllegalMemberException.class)
+ public void testGetShortForClassCastException() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForInt = new ExpressionResult(DataType.INT, "STRING");
+ expressionResultForInt.getShort();
+ }
+
+ @Test public void testGetStringForNull() throws FilterIllegalMemberException {
+ assertNull(expressionResult.getString());
+ }
+
+ @Test public void testGetStringForString() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForString = new ExpressionResult(DataType.STRING, "5");
+ String actualValue = expressionResultForString.getString();
+ String expectedValue = "5";
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test public void testGetStringForException() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForString = new ExpressionResult(DataType.INT, "5");
+ String actualValue = expressionResultForString.getString();
+ String expectedValue = "5";
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test public void testGetDoubleForNull() throws FilterIllegalMemberException {
+ assertNull(expressionResult.getDouble());
+ }
+
+ @Test public void testGetDoubleForString() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForString = new ExpressionResult(DataType.STRING, "5");
+ double actualValue = expressionResultForString.getDouble();
+ double expectedValue = 5;
+ assertEquals(expectedValue, actualValue, DELTA);
+ }
+
+ @Test(expected = FilterIllegalMemberException.class)
+ public void testGetDoubleForNumberFormatException() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForString = new ExpressionResult(DataType.STRING, "FOOBAR");
+ expressionResultForString.getDouble();
+ }
+
+ @Test public void testGetDoubleForInt() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForDouble = new ExpressionResult(DataType.INT, 5);
+ double actualValue = expressionResultForDouble.getDouble();
+ double expectedValue = 5.0;
+ assertEquals(expectedValue, actualValue, DELTA);
+ }
+
+ @Test public void testGetDoubleForDouble() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForDouble = new ExpressionResult(DataType.DOUBLE, 5.0);
+ double actualValue = expressionResultForDouble.getDouble();
+ double expectedValue = 5;
+ assertEquals(expectedValue, actualValue, DELTA);
+ }
+
+ @Test(expected = FilterIllegalMemberException.class)
+ public void testGetDoubleForIllegalMemberException() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForDouble = new ExpressionResult(DataType.FLOAT, 25.36);
+ expressionResultForDouble.getDouble();
+ }
+
+ @Test(expected = FilterIllegalMemberException.class)
+ public void testGetDoubleForClassCastException() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForDouble = new ExpressionResult(DataType.DOUBLE, "STRING");
+ expressionResultForDouble.getDouble();
+ }
+
+ @Test public void testGetLongForNull() throws FilterIllegalMemberException {
+ assertNull(expressionResult.getLong());
+ }
+
+ @Test public void testGetLongForString() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForString = new ExpressionResult(DataType.STRING, "5");
+ long actualValue = expressionResultForString.getLong();
+ long expectedValue = 5;
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test(expected = FilterIllegalMemberException.class)
+ public void testGetLongForNumberFormatException() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForString = new ExpressionResult(DataType.STRING, "FOOBAR");
+ expressionResultForString.getLong();
+ }
+
+ @Test(expected = FilterIllegalMemberException.class)
+ public void testGetLongForIllegalMemberException() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForLong = new ExpressionResult(DataType.FLOAT, 25.36);
+ expressionResultForLong.getLong();
+ }
+
+ @Test(expected = FilterIllegalMemberException.class)
+ public void testGetLongForClassCastException() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForLong = new ExpressionResult(DataType.LONG, "STRING");
+ expressionResultForLong.getLong();
+ }
+
+ @Test public void testGetDecimalForNull() throws FilterIllegalMemberException {
+ assertNull(expressionResult.getDecimal());
+ }
+
+ @Test public void testGetDecimalForString() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForString = new ExpressionResult(DataType.STRING, "5");
+ BigDecimal actualValue = expressionResultForString.getDecimal();
+ BigDecimal expectedValue = new BigDecimal(5.00);
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test(expected = FilterIllegalMemberException.class)
+ public void testGetDecimalForNumberFormatException() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForString = new ExpressionResult(DataType.STRING, "FOOBAR");
+ expressionResultForString.getDecimal();
+ }
+
+ @Test public void testGetDecimalForInt() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForInt = new ExpressionResult(DataType.INT, 5);
+ BigDecimal actualValue = expressionResultForInt.getDecimal();
+ BigDecimal expectedValue = new BigDecimal(5);
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test public void testGetDecimalForDouble() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForDouble = new ExpressionResult(DataType.DOUBLE, 5);
+ BigDecimal actualValue = expressionResultForDouble.getDecimal();
+ BigDecimal expectedValue = new BigDecimal(5);
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test public void testGetDecimalForDecimal() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForDecimal = new ExpressionResult(DataType.DECIMAL, 5);
+ BigDecimal actualValue = expressionResultForDecimal.getDecimal();
+ BigDecimal expectedValue = new BigDecimal(5);
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test(expected = FilterIllegalMemberException.class)
+ public void testGetDecimalForIllegalMemberException() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForFloat = new ExpressionResult(DataType.FLOAT, 25.36);
+ expressionResultForFloat.getDecimal();
+ }
+
+ @Test(expected = FilterIllegalMemberException.class)
+ public void testGetDecimalForClassCastException() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForException = new ExpressionResult(DataType.LONG, "STRING");
+ expressionResultForException.getDecimal();
+ }
+
+ @Test public void testGetTimeForNull() throws FilterIllegalMemberException {
+ assertNull(expressionResult.getTime());
+ }
+
+ @Test public void testGetTimeForString() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForString =
+ new ExpressionResult(DataType.STRING, "2016-11-07 10:15:09");
+ long actualValue = expressionResultForString.getTime();
+ long expectedValue = getTime("2016-11-07 10:15:09");
+ assertEquals(expectedValue, actualValue);
+ }
+
+ public Long getTime(String value) throws FilterIllegalMemberException {
+ if (value == null) {
+ return null;
+ }
+ SimpleDateFormat parser =
+ new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT);
+ Date dateToStr;
+ try {
+ dateToStr = parser.parse(value.toString());
+ return dateToStr.getTime();
+ } catch (ParseException e) {
+ throw new FilterIllegalMemberException("Cannot convert value to Time/Long type value");
+ }
+ }
+
+ @Test(expected = FilterIllegalMemberException.class) public void testGetTimeForParseException()
+ throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForParseException =
+ new ExpressionResult(DataType.STRING, "FOOBAR");
+ expressionResultForParseException.getTime();
+ }
+
+ @Test(expected = FilterIllegalMemberException.class)
+ public void testGetTimeForIllegalMemberException() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForFloat = new ExpressionResult(DataType.FLOAT, 25.36);
+ expressionResultForFloat.getTime();
+ }
+
+ @Test(expected = FilterIllegalMemberException.class)
+ public void testGetTimeForClassCastException() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForException = new ExpressionResult(DataType.LONG, "STRING");
+ expressionResultForException.getTime();
+ }
+
+ @Test public void testGetBooleanForNull() throws FilterIllegalMemberException {
+ assertNull(expressionResult.getBoolean());
+ }
+
+ @Test public void testGetBooleanForString() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForString = new ExpressionResult(DataType.STRING, "true");
+ boolean actualValue = expressionResultForString.getBoolean();
+ boolean expectedValue = true;
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test public void testGetBooleanForBoolean() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForString = new ExpressionResult(DataType.BOOLEAN, "true");
+ boolean actualValue = expressionResultForString.getBoolean();
+ boolean expectedValue = true;
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test(expected = FilterIllegalMemberException.class)
+ public void testGetBooleanForIllegalMemberException() throws FilterIllegalMemberException {
+ ExpressionResult expressionResultForFloat = new ExpressionResult(DataType.FLOAT, 25.36);
+ expressionResultForFloat.getBoolean();
+ }
+
+ @Test public void testGetList() {
+ ExpressionResult obj = new ExpressionResult(DataType.STRING, "test");
+ List<ExpressionResult> actualValue = obj.getList();
+ List<ExpressionResult> expected = new ArrayList<>();
+ expected.add(0, obj);
+ assertEquals(expected, actualValue);
+ }
+
+ @Test public void testHashCode() {
+ ExpressionResult obj = new ExpressionResult(DataType.INT, 1);
+ int actualValue = obj.hashCode();
+ int expectedValue = 80;
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test public void testHashCodeForNull() {
+ ExpressionResult obj = new ExpressionResult(null);
+ int actualValue = obj.hashCode();
+ int expectedValue = 31;
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test public void testHashCodeForNul() {
+ ExpressionResult obj = new ExpressionResult(DataType.DOUBLE, null);
+ int actualValue = obj.hashCode();
+ int expectedValue = 31;
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test public void testEqualsForObjNOtInstanceOfExpressionResult() {
+ ExpressionResult expressionResult = new ExpressionResult(DataType.INT, 5);
+ List obj = new ArrayList();
+ obj.add(0, 0);
+ obj.add(1, 1);
+ boolean result = expressionResult.equals(obj);
+ assertFalse(result);
+ }
+
+ @Test public void testEqualsForString() {
+ ExpressionResult expressionResult = new ExpressionResult(DataType.STRING, "5");
+ ExpressionResult objToCompare = new ExpressionResult(DataType.STRING, "6");
+ boolean result = expressionResult.equals(objToCompare);
+ assertFalse(result);
+ }
+
+ @Test public void testEqualsForShort() {
+ ExpressionResult expressionResult = new ExpressionResult(DataType.SHORT, 5);
+ ExpressionResult objToCompare = new ExpressionResult(DataType.SHORT, 6);
+ boolean result = expressionResult.equals(objToCompare);
+ assertFalse(result);
+ }
+
+ @Test public void testEqualsForInt() {
+ ExpressionResult expressionResult = new ExpressionResult(DataType.INT, 5);
+ ExpressionResult objToCompare = new ExpressionResult(DataType.INT, 6);
+ boolean result = expressionResult.equals(objToCompare);
+ assertFalse(result);
+ }
+
+ @Test public void testEqualsForDecimal() {
+ ExpressionResult expressionResult = new ExpressionResult(DataType.DECIMAL, 5);
+ ExpressionResult objToCompare = new ExpressionResult(DataType.DECIMAL, 6);
+ boolean result = expressionResult.equals(objToCompare);
+ assertFalse(result);
+ }
+
+ @Test public void testEqualsForDouble() {
+ ExpressionResult expressionResult = new ExpressionResult(DataType.DOUBLE, 5.89);
+ ExpressionResult objToCompare = new ExpressionResult(DataType.DOUBLE, 6.90);
+ boolean result = expressionResult.equals(objToCompare);
+ assertFalse(result);
+ }
+
+ @Test public void testEqualsForException() {
+ ExpressionResult expressionResult =
+ new ExpressionResult(DataType.TIMESTAMP, "2016-11-07 10:15:09");
+ ExpressionResult objToCompare = new ExpressionResult(DataType.DOUBLE, "2016-11-06 10:15:09");
+ boolean result = expressionResult.equals(objToCompare);
+ assertFalse(result);
+ }
+
+ @Test public void testEqualsForLongAndTimeStamp() {
+ ExpressionResult expressionResult =
+ new ExpressionResult(DataType.TIMESTAMP, new Long(11111111111111111L));
+ ExpressionResult objToCompare =
+ new ExpressionResult(DataType.LONG, new Long(11111111111111111L));
+ boolean result = expressionResult.equals(objToCompare);
+ assertTrue(result);
+ }
+
+ @Test public void compareToForInt() {
+ ExpressionResult obj = new ExpressionResult(DataType.INT, 5);
+ ExpressionResult expressionResult = new ExpressionResult(DataType.INT, 6);
+ int actualValue = expressionResult.compareTo(obj);
+ int expectedValue = 1;
+ assertEquals(expectedValue, actualValue);
+
+ }
+
+ @Test public void compareToForDecimal() {
+ ExpressionResult obj = new ExpressionResult(DataType.DECIMAL, 5);
+ ExpressionResult expressionResult = new ExpressionResult(DataType.DECIMAL, 6);
+ int actualValue = expressionResult.compareTo(obj);
+ int expectedValue = 1;
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test public void compareToForException() {
+ ExpressionResult obj = new ExpressionResult(DataType.INT, 5);
+ ExpressionResult expressionResult = new ExpressionResult(DataType.DECIMAL, 6);
+ int actualValue = expressionResult.compareTo(obj);
+ int expectedValue = -1;
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test public void compareToForTimestamp() {
+ ExpressionResult obj = new ExpressionResult(DataType.TIMESTAMP, "2016-11-07 10:15:09");
+ ExpressionResult expressionResult =
+ new ExpressionResult(DataType.TIMESTAMP, "2016-11-07 10:15:09");
+ int actualValue = expressionResult.compareTo(obj);
+ int expectedValue = 0;
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test public void testGetListAsString() throws FilterIllegalMemberException {
+ ExpressionResult obj = new ExpressionResult(DataType.INT, 6);
+ List<String> actualValue = obj.getListAsString();
+ List<String> expectedValue = new ArrayList<>();
+ expectedValue.add("6");
+ assertThat(actualValue, is(equalTo(expectedValue)));
+ }
+
+ @Test public void compareToForString() {
+ ExpressionResult obj = new ExpressionResult(DataType.STRING, "2016");
+ ExpressionResult expressionResult = new ExpressionResult(DataType.STRING, "2016");
+ int actualValue = expressionResult.compareTo(obj);
+ int expectedValue = 0;
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test public void testIsNullForNull() {
+ ExpressionResult obj = new ExpressionResult(DataType.STRING, null);
+ boolean actualValue = obj.isNull();
+ assertTrue(actualValue);
+ }
+
+ @Test public void testIsNullForNotNull() {
+ ExpressionResult obj = new ExpressionResult(DataType.STRING, "test");
+ boolean actualValue = obj.isNull();
+ assertFalse(actualValue);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/scan/expression/LiteralExpressionTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/LiteralExpressionTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/LiteralExpressionTest.java
new file mode 100644
index 0000000..00a2a18
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/LiteralExpressionTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.scan.filter.intf.RowImpl;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class LiteralExpressionTest {
+
+ private LiteralExpression literalExpression;
+
+ @Before public void setUp() {
+ String value = "testing";
+ literalExpression = new LiteralExpression(value, DataType.STRING);
+ }
+
+ @Test public void testEvaluate() {
+ RowImpl rowImpl = new RowImpl();
+ rowImpl.setValues(new String[] { "testing" });
+ ExpressionResult expectedResult = new ExpressionResult(DataType.STRING, "testing");
+ assertEquals(expectedResult, literalExpression.evaluate(rowImpl));
+ }
+
+ @Test public void testGetExpressionResult() {
+ RowImpl rowImpl = new RowImpl();
+ rowImpl.setValues(new String[] { "testing" });
+ literalExpression.evaluate(rowImpl);
+ ExpressionResult expectedResult = new ExpressionResult(DataType.STRING, "testing");
+ assertEquals(expectedResult, literalExpression.evaluate(rowImpl));
+ }
+
+ @Test public void testGetString() {
+ String actualValue = literalExpression.getString();
+ String expectedValue = "LiteralExpression(testing)";
+ assertEquals(expectedValue, actualValue);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java
new file mode 100644
index 0000000..3893a07
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpressionUnitTest.java
@@ -0,0 +1,315 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.expression.conditional;
+
+import java.math.BigDecimal;
+import java.sql.Timestamp;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.RowImpl;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.apache.spark.sql.types.Decimal;
+import org.junit.Test;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+
+public class EqualToExpressionUnitTest {
+
+ static EqualToExpression equalToExpression;
+
+ @Test public void testForEqualToExpressionWithGetString() throws Exception {
+ ColumnExpression right = new ColumnExpression("name", DataType.STRING);
+ right.setColIndex(0);
+ equalToExpression = new EqualToExpression(right, right);
+ String expected_result = "EqualTo(ColumnExpression(name),ColumnExpression(name))";
+ String result = equalToExpression.getString();
+ assertEquals(expected_result, result);
+ }
+
+ @Test public void testEvaluateForEqualToExpressionWithShortDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
+ right.setColIndex(0);
+ equalToExpression = new EqualToExpression(right, right);
+ RowImpl value = new RowImpl();
+ Short[] row = { 15 };
+ Object objectRow[] = { row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Short getShort() {
+ return 15;
+ }
+ };
+
+ ExpressionResult result = equalToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+
+ }
+
+ @Test public void testEvaluateForEqualToExpressionWithStringDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("name", DataType.STRING);
+ right.setColIndex(0);
+ equalToExpression = new EqualToExpression(right, right);
+ RowImpl value = new RowImpl();
+ String[] row = { "String1" };
+ Object objectRow[] = { row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public String getString() {
+ return "String1";
+ }
+ };
+
+ ExpressionResult result = equalToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+
+ }
+
+ @Test public void testEvaluateForEqualToExpressionWithIntDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("number", DataType.INT);
+ right.setColIndex(0);
+ equalToExpression = new EqualToExpression(right, right);
+ RowImpl value = new RowImpl();
+ Integer[] row = { 14 };
+ Object objectRow[] = { row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Integer getInt() {
+ return 14;
+ }
+ };
+
+ ExpressionResult result = equalToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForEqualToExpressionWithDoubleDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("contact", DataType.DOUBLE);
+ right.setColIndex(0);
+ equalToExpression = new EqualToExpression(right, right);
+ RowImpl value = new RowImpl();
+ Double[] row = { 44D };
+ Object objectRow[] = { row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Double getDouble() {
+ return 44D;
+ }
+ };
+
+ ExpressionResult result = equalToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForEqualToExpressionWithLongDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("contact", DataType.LONG);
+ right.setColIndex(0);
+ equalToExpression = new EqualToExpression(right, right);
+ RowImpl value = new RowImpl();
+ Long[] row = { 1234567654321L };
+ Object objectRow[] = { row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Long getLong() {
+ return 1234567654321L;
+ }
+ };
+
+ ExpressionResult result = equalToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForEqualToExpressionWithTimestampDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ try {
+ ColumnExpression right = new ColumnExpression("timestamp", DataType.TIMESTAMP);
+ right.setColIndex(0);
+ equalToExpression = new EqualToExpression(right, right);
+ RowImpl value = new RowImpl();
+ DateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy");
+ Date date = dateFormat.parse("23/09/2007");
+ long time = date.getTime();
+ Timestamp[] row = { new Timestamp(time) };
+ Object objectRow[] = { row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Long getTime() {
+ return 18465213000000L;
+ }
+ };
+
+ ExpressionResult result = equalToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ } catch (ParseException e) {
+ System.out.println("Error while parsing " + e.getMessage());
+ }
+ }
+
+ @Test(expected = FilterUnsupportedException.class) public void testForEqualToExpressionForDefaultCase()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("contact", DataType.BOOLEAN);
+ right.setColIndex(0);
+ equalToExpression = new EqualToExpression(right, right);
+ RowImpl value = new RowImpl();
+ Boolean[] row = { true };
+ Object objectRow[] = { row };
+ value.setValues(objectRow);
+ ExpressionResult result = equalToExpression.evaluate(value);
+ }
+
+ @Test public void testEvaluateForEqualToExpressionWithBooleanParameter()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
+ right.setColIndex(0);
+ equalToExpression = new EqualToExpression(right, right, true);
+ RowImpl value = new RowImpl();
+ Short[] row = { 15 };
+ Object objectRow[] = { row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Short getShort() {
+ return 15;
+ }
+ };
+
+ ExpressionResult result = equalToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+
+ }
+
+ @Test public void testEvaluateForEqualToExpressionWithLeftAndRightDifferentDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("name", DataType.STRING);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("number", DataType.INT);
+ right.setColIndex(0);
+ equalToExpression = new EqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ String[] row1 = { "String1" };
+ Integer[] row = { 14 };
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Integer getInt() {
+ return 14;
+ }
+ };
+
+ ExpressionResult result = equalToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForEqualToExpressionWithIsNullReturnFalse()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
+ right.setColIndex(0);
+ equalToExpression = new EqualToExpression(right, right);
+ RowImpl value = new RowImpl();
+ Short[] row = { 15 };
+ Object objectRow[] = { row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public boolean isNull() {
+ return true;
+ }
+ };
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Short getShort() {
+ return 15;
+ }
+ };
+
+ ExpressionResult result = equalToExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+
+ }
+
+ @Test public void testEvaluateForEqualToExpressionWithNullWhileCreatingObject()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
+ right.setColIndex(0);
+ equalToExpression = new EqualToExpression(right, right, true);
+ RowImpl value = new RowImpl();
+ Short[] row = { 15 };
+ Object objectRow[] = { row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public boolean isNull() {
+ return true;
+ }
+ };
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Short getShort() {
+ return 15;
+ }
+ };
+
+ ExpressionResult result = equalToExpression.evaluate(value);
+ assertEquals(DataType.BOOLEAN, result.getDataType());
+
+ }
+
+ @Test public void testEvaluateForEqualToExpressionWithDecimalDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("contact", DataType.DECIMAL);
+ right.setColIndex(0);
+ equalToExpression = new EqualToExpression(right, right);
+ RowImpl value = new RowImpl();
+ Decimal[] row = new Decimal[] { Decimal.apply(12345.0) };
+ Object objectRow[] = { row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public BigDecimal getDecimal() {
+ return new BigDecimal(12345.0);
+ }
+ };
+
+ ExpressionResult result = equalToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java
new file mode 100644
index 0000000..88e3729
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.expression.conditional;
+
+import java.math.BigDecimal;
+import java.sql.Timestamp;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.RowImpl;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.apache.spark.sql.types.Decimal;
+import org.junit.Test;
+
+import static junit.framework.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class GreaterThanEqualToExpressionUnitTest {
+
+ static GreaterThanEqualToExpression greaterThanEqualToExpression;
+
+ @Test public void testEvaluateForGreaterThanEqualToExpressionWithBothStringISSame()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
+ right.setColIndex(1);
+ greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ String[] row = { "string1" };
+ String[] row1 = { "string1" };
+ Object objectRow[] = { row, row1 };
+
+ new MockUp<ExpressionResult>() {
+ @Mock public String getString() {
+ return "string1";
+ }
+ };
+ value.setValues(objectRow);
+ ExpressionResult result = greaterThanEqualToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForGreaterThanEqualToExpressionWithShortDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("id", DataType.SHORT);
+ left.setColIndex(1);
+ greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ Short[] row = { 15 };
+ Short[] row1 = { 16 };
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Short getShort() {
+ return 16;
+ }
+ };
+
+ ExpressionResult result = greaterThanEqualToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+
+ }
+
+ @Test public void testEvaluateForGreaterThanEqualToExpressionWithIntDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("right_number", DataType.INT);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("left_number", DataType.INT);
+ left.setColIndex(1);
+ greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ Integer[] row = { 140 };
+ Integer[] row1 = { 145 };
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Integer getInt() {
+ return 145;
+ }
+ };
+
+ ExpressionResult result = greaterThanEqualToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForGreaterThanEqualToExpressionWithDoubleDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("right_contact", DataType.DOUBLE);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("left_contact", DataType.DOUBLE);
+ left.setColIndex(1);
+ greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ Double[] row = { 44D };
+ Double[] row1 = { 45D };
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Double getDouble() {
+ return 45D;
+ }
+ };
+
+ ExpressionResult result = greaterThanEqualToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForGreaterThanEqualToExpressionWithLongDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("contact", DataType.LONG);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("contact", DataType.LONG);
+ left.setColIndex(1);
+ greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ Long[] row = { 1234567654321L };
+ Long[] row1 = { 1234567654321L };
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Long getLong() {
+ return 1234567654321L;
+ }
+ };
+
+ ExpressionResult result = greaterThanEqualToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForGreaterThanEqualToExpressionWithTimestampDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ try {
+ ColumnExpression left = new ColumnExpression("timestamp", DataType.TIMESTAMP);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("timestamp", DataType.TIMESTAMP);
+ right.setColIndex(1);
+ greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ DateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy");
+ Date date = dateFormat.parse("23/09/2007");
+ long time = date.getTime();
+ Timestamp[] row = { new Timestamp(time) };
+ Timestamp[] row1 = { new Timestamp(time) };
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Long getTime() {
+ return 18465213000000L;
+ }
+ };
+
+ ExpressionResult result = greaterThanEqualToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ } catch (ParseException e) {
+ System.out.println("Error while parsing " + e.getMessage());
+ }
+ }
+
+ @Test(expected = FilterUnsupportedException.class) public void testForGreaterThanEqualToExpressionWithDefaultCase()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("contact", DataType.BOOLEAN);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("contact", DataType.BOOLEAN);
+ left.setColIndex(1);
+ greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ Boolean[] row = { true };
+ Object objectRow[] = { row, row };
+ value.setValues(objectRow);
+ greaterThanEqualToExpression.evaluate(value);
+ }
+
+ @Test public void testEvaluateForGreaterThanEqualToExpressionWithDecimalDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("contact", DataType.DECIMAL);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("contact", DataType.DECIMAL);
+ left.setColIndex(1);
+ greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ Decimal[] row = new Decimal[] { Decimal.apply(12345.0) };
+ Object objectRow[] = { row, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public BigDecimal getDecimal() {
+ return new BigDecimal(12345.0);
+ }
+ };
+
+ ExpressionResult result = greaterThanEqualToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForGreaterThanEqualToExpressionWithIsNullReturnTrue()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("id", DataType.SHORT);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
+ right.setColIndex(1);
+ greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ Short[] row = { 15 };
+ Object objectRow[] = { row, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public boolean isNull() {
+ return true;
+ }
+ };
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Short getShort() {
+ return 15;
+ }
+ };
+
+ ExpressionResult result = greaterThanEqualToExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+
+ }
+
+ @Test public void testEvaluateForGreaterThanEqualToExpressionWithLeftAndRightDifferentDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("name", DataType.STRING);
+ left.setColIndex(1);
+ ColumnExpression right = new ColumnExpression("number", DataType.INT);
+ right.setColIndex(0);
+ greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ String[] row1 = { "String1" };
+ Integer[] row = { 14 };
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Integer getInt() {
+ return 14;
+ }
+ };
+
+ ExpressionResult result = greaterThanEqualToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testForGreaterThanEqualToExpressionWithGetString() throws Exception {
+ ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
+ right.setColIndex(1);
+ greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
+ String expected_result = "GreaterThanEqualTo(ColumnExpression(left_name),ColumnExpression(right_name))";
+ String result = greaterThanEqualToExpression.getString();
+ assertEquals(expected_result, result);
+ }
+
+}
+
[15/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/scan/filter/FilterUtil.java
deleted file mode 100644
index aa5681d..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/FilterUtil.java
+++ /dev/null
@@ -1,1353 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.filter;
-
-import java.io.IOException;
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.cache.Cache;
-import org.apache.carbondata.core.cache.CacheProvider;
-import org.apache.carbondata.core.cache.CacheType;
-import org.apache.carbondata.core.cache.dictionary.Dictionary;
-import org.apache.carbondata.core.cache.dictionary.DictionaryChunksWrapper;
-import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
-import org.apache.carbondata.core.cache.dictionary.ForwardDictionary;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.datastore.IndexKey;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.core.metadata.Encoding;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
-import org.apache.carbondata.core.util.ByteUtil;
-import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.LiteralExpression;
-import org.apache.carbondata.scan.expression.conditional.ListExpression;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.executer.AndFilterExecuterImpl;
-import org.apache.carbondata.scan.filter.executer.DimColumnExecuterFilterInfo;
-import org.apache.carbondata.scan.filter.executer.ExcludeColGroupFilterExecuterImpl;
-import org.apache.carbondata.scan.filter.executer.ExcludeFilterExecuterImpl;
-import org.apache.carbondata.scan.filter.executer.FilterExecuter;
-import org.apache.carbondata.scan.filter.executer.IncludeColGroupFilterExecuterImpl;
-import org.apache.carbondata.scan.filter.executer.IncludeFilterExecuterImpl;
-import org.apache.carbondata.scan.filter.executer.OrFilterExecuterImpl;
-import org.apache.carbondata.scan.filter.executer.RestructureFilterExecuterImpl;
-import org.apache.carbondata.scan.filter.executer.RowLevelFilterExecuterImpl;
-import org.apache.carbondata.scan.filter.executer.RowLevelRangeTypeExecuterFacory;
-import org.apache.carbondata.scan.filter.intf.ExpressionType;
-import org.apache.carbondata.scan.filter.intf.FilterExecuterType;
-import org.apache.carbondata.scan.filter.intf.RowImpl;
-import org.apache.carbondata.scan.filter.intf.RowIntf;
-import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
-import org.apache.carbondata.scan.filter.resolver.RowLevelFilterResolverImpl;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-
-public final class FilterUtil {
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(FilterUtil.class.getName());
-
- private FilterUtil() {
-
- }
-
- /**
- * Pattern used : Visitor Pattern
- * Method will create filter executer tree based on the filter resolved tree,
- * in this algorithm based on the resolver instance the executers will be visited
- * and the resolved surrogates will be converted to keys
- *
- * @param filterExpressionResolverTree
- * @param segmentProperties
- * @return FilterExecuter instance
- */
- private static FilterExecuter createFilterExecuterTree(
- FilterResolverIntf filterExpressionResolverTree, SegmentProperties segmentProperties,
- Map<Integer, GenericQueryType> complexDimensionInfoMap) {
- FilterExecuterType filterExecuterType = filterExpressionResolverTree.getFilterExecuterType();
- if (null != filterExecuterType) {
- switch (filterExecuterType) {
- case INCLUDE:
- return getIncludeFilterExecuter(
- filterExpressionResolverTree.getDimColResolvedFilterInfo(), segmentProperties);
- case EXCLUDE:
- return getExcludeFilterExecuter(
- filterExpressionResolverTree.getDimColResolvedFilterInfo(), segmentProperties);
- case OR:
- return new OrFilterExecuterImpl(
- createFilterExecuterTree(filterExpressionResolverTree.getLeft(), segmentProperties,
- complexDimensionInfoMap),
- createFilterExecuterTree(filterExpressionResolverTree.getRight(), segmentProperties,
- complexDimensionInfoMap));
- case AND:
- return new AndFilterExecuterImpl(
- createFilterExecuterTree(filterExpressionResolverTree.getLeft(), segmentProperties,
- complexDimensionInfoMap),
- createFilterExecuterTree(filterExpressionResolverTree.getRight(), segmentProperties,
- complexDimensionInfoMap));
- case RESTRUCTURE:
- return new RestructureFilterExecuterImpl(
- filterExpressionResolverTree.getDimColResolvedFilterInfo(),
- segmentProperties);
- case ROWLEVEL_LESSTHAN:
- case ROWLEVEL_LESSTHAN_EQUALTO:
- case ROWLEVEL_GREATERTHAN_EQUALTO:
- case ROWLEVEL_GREATERTHAN:
- return RowLevelRangeTypeExecuterFacory
- .getRowLevelRangeTypeExecuter(filterExecuterType, filterExpressionResolverTree,
- segmentProperties);
- case ROWLEVEL:
- default:
- return new RowLevelFilterExecuterImpl(
- ((RowLevelFilterResolverImpl) filterExpressionResolverTree)
- .getDimColEvaluatorInfoList(),
- ((RowLevelFilterResolverImpl) filterExpressionResolverTree)
- .getMsrColEvalutorInfoList(),
- ((RowLevelFilterResolverImpl) filterExpressionResolverTree).getFilterExpresion(),
- ((RowLevelFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(),
- segmentProperties, complexDimensionInfoMap);
-
- }
- }
- return new RowLevelFilterExecuterImpl(
- ((RowLevelFilterResolverImpl) filterExpressionResolverTree).getDimColEvaluatorInfoList(),
- ((RowLevelFilterResolverImpl) filterExpressionResolverTree).getMsrColEvalutorInfoList(),
- ((RowLevelFilterResolverImpl) filterExpressionResolverTree).getFilterExpresion(),
- ((RowLevelFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(),
- segmentProperties, complexDimensionInfoMap);
-
- }
-
- /**
- * It gives filter executer based on columnar or column group
- *
- * @param dimColResolvedFilterInfo
- * @param segmentProperties
- * @return
- */
- private static FilterExecuter getIncludeFilterExecuter(
- DimColumnResolvedFilterInfo dimColResolvedFilterInfo, SegmentProperties segmentProperties) {
-
- if (dimColResolvedFilterInfo.getDimension().isColumnar()) {
- return new IncludeFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
- } else {
- return new IncludeColGroupFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
- }
- }
-
- /**
- * It gives filter executer based on columnar or column group
- *
- * @param dimColResolvedFilterInfo
- * @param segmentProperties
- * @return
- */
- private static FilterExecuter getExcludeFilterExecuter(
- DimColumnResolvedFilterInfo dimColResolvedFilterInfo, SegmentProperties segmentProperties) {
-
- if (dimColResolvedFilterInfo.getDimension().isColumnar()) {
- return new ExcludeFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
- } else {
- return new ExcludeColGroupFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
- }
- }
-
- /**
- * This method will check if a given expression contains a column expression
- * recursively.
- *
- * @return
- */
- public static boolean checkIfExpressionContainsColumn(Expression expression) {
- if (expression instanceof ColumnExpression) {
- return true;
- }
- for (Expression child : expression.getChildren()) {
- if (checkIfExpressionContainsColumn(child)) {
- return true;
- }
- }
-
- return false;
- }
-
- /**
- * This method will check if a given expression contains a column expression
- * recursively.
- *
- * @return
- */
- public static boolean checkIfLeftExpressionRequireEvaluation(Expression expression) {
- if (expression.getFilterExpressionType() == ExpressionType.UNKNOWN
- || !(expression instanceof ColumnExpression)) {
- return true;
- }
- for (Expression child : expression.getChildren()) {
- if (checkIfLeftExpressionRequireEvaluation(child)) {
- return true;
- }
- }
- return false;
- }
-
- /**
- * This method will check if a given literal expression is not a timestamp datatype
- * recursively.
- *
- * @return
- */
- public static boolean checkIfDataTypeNotTimeStamp(Expression expression) {
- if (expression.getFilterExpressionType() == ExpressionType.LITERAL) {
- DataType dataType = ((LiteralExpression) expression).getLiteralExpDataType();
- if (!(dataType == DataType.TIMESTAMP || dataType == DataType.DATE)) {
- return true;
- }
- }
- for (Expression child : expression.getChildren()) {
- if (checkIfDataTypeNotTimeStamp(child)) {
- return true;
- }
- }
- return false;
- }
- /**
- * This method will check if a given expression contains a column expression
- * recursively.
- *
- * @return
- */
- public static boolean checkIfRightExpressionRequireEvaluation(Expression expression) {
- if (expression.getFilterExpressionType() == ExpressionType.UNKNOWN
- || !(expression instanceof LiteralExpression) && !(expression instanceof ListExpression)) {
- return true;
- }
- for (Expression child : expression.getChildren()) {
- if (checkIfRightExpressionRequireEvaluation(child)) {
- return true;
- }
- }
- return false;
- }
-
- /**
- * method will get the masked keys based on the keys generated from surrogates.
- *
- * @param ranges
- * @param key
- * @return byte[]
- */
- private static byte[] getMaskedKey(int[] ranges, byte[] key) {
- byte[] maskkey = new byte[ranges.length];
-
- for (int i = 0; i < maskkey.length; i++) {
- maskkey[i] = key[ranges[i]];
- }
- return maskkey;
- }
-
- /**
- * This method will return the ranges for the masked Bytes based on the key
- * Generator.
- *
- * @param queryDimensionsOrdinal
- * @param generator
- * @return
- */
- private static int[] getRangesForMaskedByte(int queryDimensionsOrdinal, KeyGenerator generator) {
- Set<Integer> integers = new TreeSet<Integer>();
- int[] range = generator.getKeyByteOffsets(queryDimensionsOrdinal);
- for (int j = range[0]; j <= range[1]; j++) {
- integers.add(j);
- }
-
- int[] byteIndexs = new int[integers.size()];
- int j = 0;
- for (Iterator<Integer> iterator = integers.iterator(); iterator.hasNext(); ) {
- Integer integer = iterator.next();
- byteIndexs[j++] = integer;
- }
- return byteIndexs;
- }
-
- /**
- * This method will get the no dictionary data based on filters and same
- * will be in DimColumnFilterInfo
- *
- * @param evaluateResultListFinal
- * @param isIncludeFilter
- * @return DimColumnFilterInfo
- */
- public static DimColumnFilterInfo getNoDictionaryValKeyMemberForFilter(
- List<String> evaluateResultListFinal, boolean isIncludeFilter) {
- List<byte[]> filterValuesList = new ArrayList<byte[]>(20);
- for (String result : evaluateResultListFinal) {
- filterValuesList.add(result.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
- }
-
- Comparator<byte[]> filterNoDictValueComaparator = new Comparator<byte[]>() {
-
- @Override public int compare(byte[] filterMember1, byte[] filterMember2) {
- // TODO Auto-generated method stub
- return ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterMember1, filterMember2);
- }
-
- };
- Collections.sort(filterValuesList, filterNoDictValueComaparator);
- DimColumnFilterInfo columnFilterInfo = null;
- if (filterValuesList.size() > 0) {
- columnFilterInfo = new DimColumnFilterInfo();
- columnFilterInfo.setIncludeFilter(isIncludeFilter);
- columnFilterInfo.setFilterListForNoDictionaryCols(filterValuesList);
-
- }
- return columnFilterInfo;
- }
-
- /**
- * Method will prepare the dimfilterinfo instance by resolving the filter
- * expression value to its respective surrogates.
- *
- * @param tableIdentifier
- * @param columnExpression
- * @param evaluateResultList
- * @param isIncludeFilter
- * @return
- * @throws IOException
- */
- public static DimColumnFilterInfo getFilterValues(AbsoluteTableIdentifier tableIdentifier,
- ColumnExpression columnExpression, List<String> evaluateResultList, boolean isIncludeFilter)
- throws IOException {
- Dictionary forwardDictionary = null;
- try {
- // Reading the dictionary value from cache.
- forwardDictionary =
- getForwardDictionaryCache(tableIdentifier, columnExpression.getDimension());
- return getFilterValues(columnExpression, evaluateResultList, forwardDictionary,
- isIncludeFilter);
- } finally {
- CarbonUtil.clearDictionaryCache(forwardDictionary);
- }
- }
-
- /**
- * Method will prepare the dimfilterinfo instance by resolving the filter
- * expression value to its respective surrogates.
- *
- * @param columnExpression
- * @param evaluateResultList
- * @param forwardDictionary
- * @param isIncludeFilter
- * @return
- */
- private static DimColumnFilterInfo getFilterValues(ColumnExpression columnExpression,
- List<String> evaluateResultList, Dictionary forwardDictionary, boolean isIncludeFilter) {
- sortFilterModelMembers(columnExpression, evaluateResultList);
- List<Integer> surrogates =
- new ArrayList<Integer>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- // Reading the dictionary value from cache.
- getDictionaryValue(evaluateResultList, forwardDictionary, surrogates);
- Collections.sort(surrogates);
- DimColumnFilterInfo columnFilterInfo = null;
- if (surrogates.size() > 0) {
- columnFilterInfo = new DimColumnFilterInfo();
- columnFilterInfo.setIncludeFilter(isIncludeFilter);
- columnFilterInfo.setFilterList(surrogates);
- }
- return columnFilterInfo;
- }
-
- /**
- * This API will get the Dictionary value for the respective filter member
- * string.
- *
- * @param evaluateResultList filter value
- * @param surrogates
- */
- private static void getDictionaryValue(List<String> evaluateResultList,
- Dictionary forwardDictionary, List<Integer> surrogates) {
- ((ForwardDictionary) forwardDictionary)
- .getSurrogateKeyByIncrementalSearch(evaluateResultList, surrogates);
- }
-
- /**
- * This method will get all the members of column from the forward dictionary
- * cache, this method will be basically used in row level filter resolver.
- *
- * @param tableIdentifier
- * @param expression
- * @param columnExpression
- * @param isIncludeFilter
- * @return DimColumnFilterInfo
- * @throws FilterUnsupportedException
- * @throws IOException
- */
- public static DimColumnFilterInfo getFilterListForAllValues(
- AbsoluteTableIdentifier tableIdentifier, Expression expression,
- final ColumnExpression columnExpression, boolean isIncludeFilter)
- throws IOException, FilterUnsupportedException {
- Dictionary forwardDictionary = null;
- List<String> evaluateResultListFinal = new ArrayList<String>(20);
- DictionaryChunksWrapper dictionaryWrapper = null;
- try {
- forwardDictionary =
- getForwardDictionaryCache(tableIdentifier, columnExpression.getDimension());
- dictionaryWrapper = forwardDictionary.getDictionaryChunks();
- while (dictionaryWrapper.hasNext()) {
- byte[] columnVal = dictionaryWrapper.next();
- try {
- RowIntf row = new RowImpl();
- String stringValue =
- new String(columnVal, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
- if (stringValue.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
- stringValue = null;
- }
- row.setValues(new Object[] { DataTypeUtil.getDataBasedOnDataType(stringValue,
- columnExpression.getCarbonColumn().getDataType()) });
- Boolean rslt = expression.evaluate(row).getBoolean();
- if (null != rslt && rslt == isIncludeFilter) {
- if (null == stringValue) {
- evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
- } else {
- evaluateResultListFinal.add(stringValue);
- }
- }
- } catch (FilterIllegalMemberException e) {
- LOGGER.debug(e.getMessage());
- }
- }
- return getFilterValues(columnExpression, evaluateResultListFinal, forwardDictionary,
- isIncludeFilter);
- } finally {
- CarbonUtil.clearDictionaryCache(forwardDictionary);
- }
- }
-
- private static void sortFilterModelMembers(final ColumnExpression columnExpression,
- List<String> evaluateResultListFinal) {
- Comparator<String> filterActualValueComaparator = new Comparator<String>() {
-
- @Override public int compare(String filterMember1, String filterMember2) {
- return compareFilterMembersBasedOnActualDataType(filterMember1, filterMember2,
- columnExpression.getDataType());
- }
-
- };
- Collections.sort(evaluateResultListFinal, filterActualValueComaparator);
- }
-
- /**
- * Method will prepare the dimfilterinfo instance by resolving the filter
- * expression value to its respective surrogates in the scenario of restructure.
- *
- * @param expression
- * @param columnExpression
- * @param defaultValues
- * @param defaultSurrogate
- * @return
- * @throws FilterUnsupportedException
- */
- public static DimColumnFilterInfo getFilterListForRS(Expression expression,
- ColumnExpression columnExpression, String defaultValues, int defaultSurrogate)
- throws FilterUnsupportedException {
- List<Integer> filterValuesList = new ArrayList<Integer>(20);
- DimColumnFilterInfo columnFilterInfo = null;
- // List<byte[]> filterValuesList = new ArrayList<byte[]>(20);
- List<String> evaluateResultListFinal = new ArrayList<String>(20);
- // KeyGenerator keyGenerator =
- // KeyGeneratorFactory.getKeyGenerator(new int[] { defaultSurrogate });
- try {
- List<ExpressionResult> evaluateResultList = expression.evaluate(null).getList();
- for (ExpressionResult result : evaluateResultList) {
- if (result.getString() == null) {
- evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
- continue;
- }
- evaluateResultListFinal.add(result.getString());
- }
-
- for (int i = 0; i < evaluateResultListFinal.size(); i++) {
- if (evaluateResultListFinal.get(i).equals(defaultValues)) {
- filterValuesList.add(defaultSurrogate);
- break;
- }
- }
- if (filterValuesList.size() > 0) {
- columnFilterInfo = new DimColumnFilterInfo();
- columnFilterInfo.setFilterList(filterValuesList);
- }
- } catch (FilterIllegalMemberException e) {
- LOGGER.audit(e.getMessage());
- }
- return columnFilterInfo;
- }
-
- /**
- * This method will get the member based on filter expression evaluation from the
- * forward dictionary cache, this method will be basically used in restructure.
- *
- * @param expression
- * @param columnExpression
- * @param defaultValues
- * @param defaultSurrogate
- * @param isIncludeFilter
- * @return
- * @throws FilterUnsupportedException
- */
- public static DimColumnFilterInfo getFilterListForAllMembersRS(Expression expression,
- ColumnExpression columnExpression, String defaultValues, int defaultSurrogate,
- boolean isIncludeFilter) throws FilterUnsupportedException {
- List<Integer> filterValuesList = new ArrayList<Integer>(20);
- List<String> evaluateResultListFinal = new ArrayList<String>(20);
- DimColumnFilterInfo columnFilterInfo = null;
-
- // KeyGenerator keyGenerator =
- // KeyGeneratorFactory.getKeyGenerator(new int[] { defaultSurrogate });
- try {
- RowIntf row = new RowImpl();
- if (defaultValues.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
- defaultValues = null;
- }
- row.setValues(new Object[] { DataTypeUtil.getDataBasedOnDataType(defaultValues,
- columnExpression.getCarbonColumn().getDataType()) });
- Boolean rslt = expression.evaluate(row).getBoolean();
- if (null != rslt && rslt == isIncludeFilter) {
- if (null == defaultValues) {
- evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
- } else {
- evaluateResultListFinal.add(defaultValues);
- }
- }
- } catch (FilterIllegalMemberException e) {
- LOGGER.audit(e.getMessage());
- }
-
- if (null == defaultValues) {
- defaultValues = CarbonCommonConstants.MEMBER_DEFAULT_VAL;
- }
- columnFilterInfo = new DimColumnFilterInfo();
- for (int i = 0; i < evaluateResultListFinal.size(); i++) {
- if (evaluateResultListFinal.get(i).equals(defaultValues)) {
- filterValuesList.add(defaultSurrogate);
- break;
- }
- }
- columnFilterInfo.setFilterList(filterValuesList);
- return columnFilterInfo;
- }
-
- /**
- * Below method will be used to covert the filter surrogate keys
- * to mdkey
- *
- * @param dimColumnFilterInfo
- * @param carbonDimension
- * @param segmentProperties
- * @return
- */
- public static byte[][] getKeyArray(DimColumnFilterInfo dimColumnFilterInfo,
- CarbonDimension carbonDimension, SegmentProperties segmentProperties) {
- if (!carbonDimension.hasEncoding(Encoding.DICTIONARY)) {
- return dimColumnFilterInfo.getNoDictionaryFilterValuesList()
- .toArray((new byte[dimColumnFilterInfo.getNoDictionaryFilterValuesList().size()][]));
- }
- KeyGenerator blockLevelKeyGenerator = segmentProperties.getDimensionKeyGenerator();
- int[] dimColumnsCardinality = segmentProperties.getDimColumnsCardinality();
- int[] keys = new int[blockLevelKeyGenerator.getDimCount()];
- List<byte[]> filterValuesList = new ArrayList<byte[]>(20);
- Arrays.fill(keys, 0);
- int[] rangesForMaskedByte =
- getRangesForMaskedByte((carbonDimension.getKeyOrdinal()), blockLevelKeyGenerator);
- if (null != dimColumnFilterInfo) {
- for (Integer surrogate : dimColumnFilterInfo.getFilterList()) {
- try {
- if (surrogate <= dimColumnsCardinality[carbonDimension.getKeyOrdinal()]) {
- keys[carbonDimension.getKeyOrdinal()] = surrogate;
- filterValuesList
- .add(getMaskedKey(rangesForMaskedByte, blockLevelKeyGenerator.generateKey(keys)));
- } else {
- break;
- }
- } catch (KeyGenException e) {
- LOGGER.error(e.getMessage());
- }
- }
-
- }
- return filterValuesList.toArray(new byte[filterValuesList.size()][]);
-
- }
-
- /**
- * The method is used to get the single dictionary key's mask key
- *
- * @param surrogate
- * @param carbonDimension
- * @param blockLevelKeyGenerator
- * @return
- */
- public static byte[] getMaskKey(int surrogate, CarbonDimension carbonDimension,
- KeyGenerator blockLevelKeyGenerator) {
-
- int[] keys = new int[blockLevelKeyGenerator.getDimCount()];
- byte[] maskedKey = null;
- Arrays.fill(keys, 0);
- int[] rangesForMaskedByte =
- getRangesForMaskedByte((carbonDimension.getKeyOrdinal()), blockLevelKeyGenerator);
- try {
- keys[carbonDimension.getKeyOrdinal()] = surrogate;
- maskedKey = getMaskedKey(rangesForMaskedByte, blockLevelKeyGenerator.generateKey(keys));
- } catch (KeyGenException e) {
- LOGGER.error(e.getMessage());
- }
- return maskedKey;
- }
-
- /**
- * Method will return the start key based on KeyGenerator for the respective
- * filter resolved instance.
- *
- * @param dimensionFilter
- * @param startKey
- * @param startKeyList
- * @return long[] start key
- */
- public static void getStartKey(Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter,
- long[] startKey, List<long[]> startKeyList) {
- for(int i = 0; i < startKey.length; i++) {
- // The min surrogate key is 1, set it as the init value for starkey of each column level
- startKey[i] = 1;
- }
- getStartKeyWithFilter(dimensionFilter, startKey, startKeyList);
- }
-
- /**
- * Algorithm for getting the start key for a filter
- * step 1: Iterate through each dimension and verify whether its not an exclude filter.
- * step 2: Intialize start key with the first filter member value present in each filter model
- * for the respective dimensions.
- * step 3: since its a no dictionary start key there will only actual value so compare
- * the first filter model value with respect to the dimension data type.
- * step 4: The least value will be considered as the start key of dimension by comparing all
- * its filter model.
- * step 5: create a byte array of start key which comprises of least filter member value of
- * all dimension and the indexes which will help to read the respective filter value.
- *
- * @param dimColResolvedFilterInfo
- * @param setOfStartKeyByteArray
- * @return
- */
- public static void getStartKeyForNoDictionaryDimension(
- DimColumnResolvedFilterInfo dimColResolvedFilterInfo,
- SortedMap<Integer, byte[]> setOfStartKeyByteArray) {
- Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter =
- dimColResolvedFilterInfo.getDimensionResolvedFilterInstance();
- // step 1
- for (Map.Entry<CarbonDimension, List<DimColumnFilterInfo>> entry : dimensionFilter.entrySet()) {
- if (!entry.getKey().hasEncoding(Encoding.DICTIONARY)) {
- List<DimColumnFilterInfo> listOfDimColFilterInfo = entry.getValue();
- if (null == listOfDimColFilterInfo) {
- continue;
- }
- boolean isExcludePresent = false;
- for (DimColumnFilterInfo info : listOfDimColFilterInfo) {
- if (!info.isIncludeFilter()) {
- isExcludePresent = true;
- }
- }
- if (isExcludePresent) {
- continue;
- }
- // step 2
- byte[] noDictionaryStartKey =
- listOfDimColFilterInfo.get(0).getNoDictionaryFilterValuesList().get(0);
- if (setOfStartKeyByteArray.isEmpty()) {
- setOfStartKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryStartKey);
- } else if (null == setOfStartKeyByteArray.get(entry.getKey().getOrdinal())) {
- setOfStartKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryStartKey);
-
- } else if (ByteUtil.UnsafeComparer.INSTANCE
- .compareTo(setOfStartKeyByteArray.get(entry.getKey().getOrdinal()),
- noDictionaryStartKey) > 0) {
- setOfStartKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryStartKey);
- }
- }
- }
- }
-
- /**
- * Algorithm for getting the end key for a filter
- * step 1: Iterate through each dimension and verify whether its not an exclude filter.
- * step 2: Initialize end key with the last filter member value present in each filter model
- * for the respective dimensions.(Already filter models are sorted)
- * step 3: since its a no dictionary end key there will only actual value so compare
- * the last filter model value with respect to the dimension data type.
- * step 4: The highest value will be considered as the end key of dimension by comparing all
- * its filter model.
- * step 5: create a byte array of end key which comprises of highest filter member value of
- * all dimension and the indexes which will help to read the respective filter value.
- *
- * @param dimColResolvedFilterInfo
- * @param setOfEndKeyByteArray
- * @return end key array
- */
- public static void getEndKeyForNoDictionaryDimension(
- DimColumnResolvedFilterInfo dimColResolvedFilterInfo,
- SortedMap<Integer, byte[]> setOfEndKeyByteArray) {
-
- Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter =
- dimColResolvedFilterInfo.getDimensionResolvedFilterInstance();
- // step 1
- for (Map.Entry<CarbonDimension, List<DimColumnFilterInfo>> entry : dimensionFilter.entrySet()) {
- if (!entry.getKey().hasEncoding(Encoding.DICTIONARY)) {
- List<DimColumnFilterInfo> listOfDimColFilterInfo = entry.getValue();
- if (null == listOfDimColFilterInfo) {
- continue;
- }
- boolean isExcludePresent = false;
- for (DimColumnFilterInfo info : listOfDimColFilterInfo) {
- if (!info.isIncludeFilter()) {
- isExcludePresent = true;
- }
- }
- if (isExcludePresent) {
- continue;
- }
- // step 2
- byte[] noDictionaryEndKey = listOfDimColFilterInfo.get(0).getNoDictionaryFilterValuesList()
- .get(listOfDimColFilterInfo.get(0).getNoDictionaryFilterValuesList().size() - 1);
- if (setOfEndKeyByteArray.isEmpty()) {
- setOfEndKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryEndKey);
- } else if (null == setOfEndKeyByteArray.get(entry.getKey().getOrdinal())) {
- setOfEndKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryEndKey);
-
- } else if (ByteUtil.UnsafeComparer.INSTANCE
- .compareTo(setOfEndKeyByteArray.get(entry.getKey().getOrdinal()), noDictionaryEndKey)
- < 0) {
- setOfEndKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryEndKey);
- }
-
- }
- }
- }
-
- /**
- * Method will pack all the byte[] to a single byte[] value by appending the
- * indexes of the byte[] value which needs to be read. this method will be mailny used
- * in case of no dictionary dimension processing for filters.
- *
- * @param noDictionaryValKeyList
- * @return packed key with its indexes added in starting and its actual values.
- */
- private static byte[] getKeyWithIndexesAndValues(List<byte[]> noDictionaryValKeyList) {
- ByteBuffer[] buffArr = new ByteBuffer[noDictionaryValKeyList.size()];
- int index = 0;
- for (byte[] singleColVal : noDictionaryValKeyList) {
- buffArr[index] = ByteBuffer.allocate(singleColVal.length);
- buffArr[index].put(singleColVal);
- buffArr[index++].rewind();
- }
- // byteBufer.
- return CarbonUtil.packByteBufferIntoSingleByteArray(buffArr);
-
- }
-
- /**
- * This method will fill the start key array with the surrogate key present
- * in filterinfo instance.
- *
- * @param dimensionFilter
- * @param startKey
- */
- private static void getStartKeyWithFilter(
- Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter, long[] startKey,
- List<long[]> startKeyList) {
- for (Map.Entry<CarbonDimension, List<DimColumnFilterInfo>> entry : dimensionFilter.entrySet()) {
- List<DimColumnFilterInfo> values = entry.getValue();
- if (null == values || !entry.getKey().hasEncoding(Encoding.DICTIONARY)) {
- continue;
- }
- boolean isExcludePresent = false;
- for (DimColumnFilterInfo info : values) {
- if (!info.isIncludeFilter()) {
- isExcludePresent = true;
- }
- }
- if (isExcludePresent) {
- continue;
- }
- for (DimColumnFilterInfo info : values) {
- if (startKey[entry.getKey().getKeyOrdinal()] < info.getFilterList().get(0)) {
- startKey[entry.getKey().getKeyOrdinal()] = info.getFilterList().get(0);
- }
- }
- long[] newStartKey = new long[startKey.length];
- System.arraycopy(startKey, 0, newStartKey, 0, startKey.length);
- startKeyList.add(newStartKey);
- }
- }
-
- public static void getEndKey(Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter,
- long[] endKey, SegmentProperties segmentProperties,
- List<long[]> endKeyList) {
-
- List<CarbonDimension> updatedDimListBasedOnKeyGenerator =
- getCarbonDimsMappedToKeyGenerator(segmentProperties.getDimensions());
- for (int i = 0; i < endKey.length; i++) {
- endKey[i] = getMaxValue(updatedDimListBasedOnKeyGenerator.get(i),
- segmentProperties.getDimColumnsCardinality());
- }
- getEndKeyWithFilter(dimensionFilter, endKey, endKeyList);
-
- }
-
- private static List<CarbonDimension> getCarbonDimsMappedToKeyGenerator(
- List<CarbonDimension> carbonDimensions) {
- List<CarbonDimension> listOfCarbonDimPartOfKeyGen =
- new ArrayList<CarbonDimension>(carbonDimensions.size());
- for (CarbonDimension carbonDim : carbonDimensions) {
- if (CarbonUtil.hasEncoding(carbonDim.getEncoder(), Encoding.DICTIONARY) || CarbonUtil
- .hasEncoding(carbonDim.getEncoder(), Encoding.DIRECT_DICTIONARY)) {
- listOfCarbonDimPartOfKeyGen.add(carbonDim);
- }
-
- }
- return listOfCarbonDimPartOfKeyGen;
- }
-
- private static void getEndKeyWithFilter(
- Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter, long[] endKey,
- List<long[]> endKeyList) {
- for (Map.Entry<CarbonDimension, List<DimColumnFilterInfo>> entry : dimensionFilter.entrySet()) {
- List<DimColumnFilterInfo> values = entry.getValue();
- if (null == values || !entry.getKey().hasEncoding(Encoding.DICTIONARY)) {
- continue;
- }
- boolean isExcludeFilterPresent = false;
- for (DimColumnFilterInfo info : values) {
- if (!info.isIncludeFilter()) {
- isExcludeFilterPresent = true;
- }
- }
- if (isExcludeFilterPresent) {
- continue;
- }
-
- for (DimColumnFilterInfo info : values) {
- if (endKey[entry.getKey().getKeyOrdinal()] > info.getFilterList()
- .get(info.getFilterList().size() - 1)) {
- endKey[entry.getKey().getKeyOrdinal()] =
- info.getFilterList().get(info.getFilterList().size() - 1);
- }
- }
- long[] newEndKey = new long[endKey.length];
- System.arraycopy(endKey, 0, newEndKey, 0, endKey.length);
- endKeyList.add(newEndKey);
- }
-
- }
-
- /**
- * This API will get the max value of surrogate key which will be used for
- * determining the end key of particular btree.
- *
- * @param dimCarinality
- */
- private static long getMaxValue(CarbonDimension carbonDimension, int[] dimCarinality) {
- // Get data from all the available slices of the table
- if (null != dimCarinality) {
- return dimCarinality[carbonDimension.getKeyOrdinal()];
- }
- return -1;
- }
-
- /**
- * @param tableIdentifier
- * @param carbonDimension
- * @return
- */
- public static Dictionary getForwardDictionaryCache(AbsoluteTableIdentifier tableIdentifier,
- CarbonDimension carbonDimension) throws IOException {
- DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier =
- new DictionaryColumnUniqueIdentifier(tableIdentifier.getCarbonTableIdentifier(),
- carbonDimension.getColumnIdentifier(), carbonDimension.getDataType());
- CacheProvider cacheProvider = CacheProvider.getInstance();
- Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache =
- cacheProvider.createCache(CacheType.FORWARD_DICTIONARY, tableIdentifier.getStorePath());
- // get the forward dictionary object
- return forwardDictionaryCache.get(dictionaryColumnUniqueIdentifier);
- }
-
- public static IndexKey createIndexKeyFromResolvedFilterVal(long[] startOrEndKey,
- KeyGenerator keyGenerator, byte[] startOrEndKeyForNoDictDimension) {
- IndexKey indexKey = null;
- try {
- indexKey =
- new IndexKey(keyGenerator.generateKey(startOrEndKey), startOrEndKeyForNoDictDimension);
- } catch (KeyGenException e) {
- LOGGER.error(e.getMessage());
- }
- return indexKey;
- }
-
- /**
- * API will create an filter executer tree based on the filter resolver
- *
- * @param filterExpressionResolverTree
- * @param segmentProperties
- * @return
- */
- public static FilterExecuter getFilterExecuterTree(
- FilterResolverIntf filterExpressionResolverTree, SegmentProperties segmentProperties,
- Map<Integer, GenericQueryType> complexDimensionInfoMap) {
- return createFilterExecuterTree(filterExpressionResolverTree, segmentProperties,
- complexDimensionInfoMap);
- }
-
- /**
- * API will prepare the Keys from the surrogates of particular filter resolver
- *
- * @param filterValues
- * @param segmentProperties
- * @param dimension
- * @param dimColumnExecuterInfo
- */
- public static void prepareKeysFromSurrogates(DimColumnFilterInfo filterValues,
- SegmentProperties segmentProperties, CarbonDimension dimension,
- DimColumnExecuterFilterInfo dimColumnExecuterInfo) {
- byte[][] keysBasedOnFilter = getKeyArray(filterValues, dimension, segmentProperties);
- dimColumnExecuterInfo.setFilterKeys(keysBasedOnFilter);
-
- }
-
- /**
- * method will create a default end key in case of no end key is been derived using existing
- * filter or in case of non filter queries.
- *
- * @param segmentProperties
- * @return
- * @throws KeyGenException
- */
- public static IndexKey prepareDefaultEndIndexKey(SegmentProperties segmentProperties)
- throws KeyGenException {
- long[] dictionarySurrogateKey =
- new long[segmentProperties.getDimensions().size() - segmentProperties
- .getNumberOfNoDictionaryDimension()];
- int index = 0;
- int[] dimColumnsCardinality = segmentProperties.getDimColumnsCardinality();
- for (int i = 0; i < dimColumnsCardinality.length; i++) {
- dictionarySurrogateKey[index++] = dimColumnsCardinality[i];
- }
- IndexKey endIndexKey;
- byte[] dictionaryendMdkey =
- segmentProperties.getDimensionKeyGenerator().generateKey(dictionarySurrogateKey);
- byte[] noDictionaryEndKeyBuffer = getNoDictionaryDefaultEndKey(segmentProperties);
- endIndexKey = new IndexKey(dictionaryendMdkey, noDictionaryEndKeyBuffer);
- return endIndexKey;
- }
-
- public static byte[] getNoDictionaryDefaultEndKey(SegmentProperties segmentProperties) {
- // in case of non filter query when no dictionary columns are present we
- // need to set the default end key, as for non filter query
- // we need to get the last
- // block of the btree so we are setting the max byte value in the end key
- ByteBuffer noDictionaryEndKeyBuffer = ByteBuffer.allocate(
- (segmentProperties.getNumberOfNoDictionaryDimension()
- * CarbonCommonConstants.SHORT_SIZE_IN_BYTE) + segmentProperties
- .getNumberOfNoDictionaryDimension());
- // end key structure will be
- //<Offset of first No Dictionary key in 2 Bytes><Offset of second No Dictionary key in 2 Bytes>
- //<Offset of n No Dictionary key in 2 Bytes><first no dictionary column value>
- // <second no dictionary column value> <N no dictionary column value>
- //example if we have 2 no dictionary column
- //<[0,4,0,5,127,127]>
- short startPoint = (short) (segmentProperties.getNumberOfNoDictionaryDimension()
- * CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
- for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
- noDictionaryEndKeyBuffer.putShort((startPoint));
- startPoint++;
- }
- for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
- noDictionaryEndKeyBuffer.put((byte) 127);
- }
- return noDictionaryEndKeyBuffer.array();
- }
-
- /**
- * method will create a default end key in case of no end key is been
- * derived using existing filter or in case of non filter queries.
- *
- * @param segmentProperties
- * @return
- * @throws KeyGenException
- */
- public static IndexKey prepareDefaultStartIndexKey(SegmentProperties segmentProperties)
- throws KeyGenException {
- IndexKey startIndexKey;
- long[] dictionarySurrogateKey =
- new long[segmentProperties.getDimensions().size() - segmentProperties
- .getNumberOfNoDictionaryDimension()];
- byte[] dictionaryStartMdkey =
- segmentProperties.getDimensionKeyGenerator().generateKey(dictionarySurrogateKey);
- byte[] noDictionaryStartKeyArray = getNoDictionaryDefaultStartKey(segmentProperties);
-
- startIndexKey = new IndexKey(dictionaryStartMdkey, noDictionaryStartKeyArray);
- return startIndexKey;
- }
-
- public static byte[] getNoDictionaryDefaultStartKey(SegmentProperties segmentProperties) {
- // in case of non filter query when no dictionary columns are present we
- // need to set the default start key, as for non filter query we need to get the first
- // block of the btree so we are setting the least byte value in the start key
- ByteBuffer noDictionaryStartKeyBuffer = ByteBuffer.allocate(
- (segmentProperties.getNumberOfNoDictionaryDimension()
- * CarbonCommonConstants.SHORT_SIZE_IN_BYTE) + segmentProperties
- .getNumberOfNoDictionaryDimension());
- // end key structure will be
- //<Offset of first No Dictionary key in 2 Bytes><Offset of second No Dictionary key in 2 Bytes>
- //<Offset of n No Dictionary key in 2 Bytes><first no dictionary column value>
- // <second no dictionary column value> <N no dictionary column value>
- //example if we have 2 no dictionary column
- //<[0,4,0,5,0,0]>
- short startPoint = (short) (segmentProperties.getNumberOfNoDictionaryDimension()
- * CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
- for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
- noDictionaryStartKeyBuffer.putShort((startPoint));
- startPoint++;
- }
- for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
- noDictionaryStartKeyBuffer.put((byte) 0);
- }
- return noDictionaryStartKeyBuffer.array();
- }
-
- public static int compareFilterKeyBasedOnDataType(String dictionaryVal, String memberVal,
- DataType dataType) {
- try {
- switch (dataType) {
- case SHORT:
- return Short.compare((Short.parseShort(dictionaryVal)), (Short.parseShort(memberVal)));
- case INT:
- return Integer.compare((Integer.parseInt(dictionaryVal)), (Integer.parseInt(memberVal)));
- case DOUBLE:
- return Double
- .compare((Double.parseDouble(dictionaryVal)), (Double.parseDouble(memberVal)));
- case LONG:
- return Long.compare((Long.parseLong(dictionaryVal)), (Long.parseLong(memberVal)));
- case BOOLEAN:
- return Boolean
- .compare((Boolean.parseBoolean(dictionaryVal)), (Boolean.parseBoolean(memberVal)));
- case DATE:
- case TIMESTAMP:
- SimpleDateFormat parser = new SimpleDateFormat(CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
- Date dateToStr;
- Date dictionaryDate;
- dateToStr = parser.parse(memberVal);
- dictionaryDate = parser.parse(dictionaryVal);
- return dictionaryDate.compareTo(dateToStr);
-
- case DECIMAL:
- java.math.BigDecimal javaDecValForDictVal = new java.math.BigDecimal(dictionaryVal);
- java.math.BigDecimal javaDecValForMemberVal = new java.math.BigDecimal(memberVal);
- return javaDecValForDictVal.compareTo(javaDecValForMemberVal);
- default:
- return -1;
- }
- } catch (Exception e) {
- return -1;
- }
- }
-
- /**
- * method will set the start and end key for as per the filter resolver tree
- * utilized visitor pattern inorder to populate the start and end key population.
- *
- * @param segmentProperties
- * @param filterResolver
- * @param listOfStartEndKeys
- */
- public static void traverseResolverTreeAndGetStartAndEndKey(SegmentProperties segmentProperties,
- FilterResolverIntf filterResolver, List<IndexKey> listOfStartEndKeys) {
- IndexKey searchStartKey = null;
- IndexKey searchEndKey = null;
- long[] startKey = new long[segmentProperties.getDimensionKeyGenerator().getDimCount()];
- long[] endKey = new long[segmentProperties.getDimensionKeyGenerator().getDimCount()];
- List<byte[]> listOfStartKeyByteArray =
- new ArrayList<byte[]>(segmentProperties.getNumberOfNoDictionaryDimension());
- List<byte[]> listOfEndKeyByteArray =
- new ArrayList<byte[]>(segmentProperties.getNumberOfNoDictionaryDimension());
- SortedMap<Integer, byte[]> setOfStartKeyByteArray = new TreeMap<Integer, byte[]>();
- SortedMap<Integer, byte[]> setOfEndKeyByteArray = new TreeMap<Integer, byte[]>();
- SortedMap<Integer, byte[]> defaultStartValues = new TreeMap<Integer, byte[]>();
- SortedMap<Integer, byte[]> defaultEndValues = new TreeMap<Integer, byte[]>();
- List<long[]> startKeyList = new ArrayList<long[]>();
- List<long[]> endKeyList = new ArrayList<long[]>();
- traverseResolverTreeAndPopulateStartAndEndKeys(filterResolver, segmentProperties, startKey,
- setOfStartKeyByteArray, endKey, setOfEndKeyByteArray,
- startKeyList, endKeyList);
- if (endKeyList.size() > 0) {
- //get the new end key from list
- for (int i = 0; i < endKey.length; i++) {
- long[] endkeyColumnLevel = new long[endKeyList.size()];
- int j = 0;
- for (long[] oneEndKey : endKeyList) {
- //get each column level end key
- endkeyColumnLevel[j++] = oneEndKey[i];
- }
- Arrays.sort(endkeyColumnLevel);
- // get the max one as end of this column level
- endKey[i] = endkeyColumnLevel[endkeyColumnLevel.length - 1];
- }
- }
-
- if (startKeyList.size() > 0) {
- //get the new start key from list
- for (int i = 0; i < startKey.length; i++) {
- long[] startkeyColumnLevel = new long[startKeyList.size()];
- int j = 0;
- for (long[] oneStartKey : startKeyList) {
- //get each column level start key
- startkeyColumnLevel[j++] = oneStartKey[i];
- }
- Arrays.sort(startkeyColumnLevel);
- // get the min - 1 as start of this column level, for example if a block contains 5,6
- // the filter is 6, but that block's start key is 5, if not -1, this block will missing.
- startKey[i] = startkeyColumnLevel[0] - 1;
- }
- }
-
- fillDefaultStartValue(defaultStartValues, segmentProperties);
- fillDefaultEndValue(defaultEndValues, segmentProperties);
- fillNullValuesStartIndexWithDefaultKeys(setOfStartKeyByteArray, segmentProperties);
- fillNullValuesEndIndexWithDefaultKeys(setOfEndKeyByteArray, segmentProperties);
- pruneStartAndEndKeys(setOfStartKeyByteArray, listOfStartKeyByteArray);
- pruneStartAndEndKeys(setOfEndKeyByteArray, listOfEndKeyByteArray);
-
- searchStartKey = FilterUtil
- .createIndexKeyFromResolvedFilterVal(startKey, segmentProperties.getDimensionKeyGenerator(),
- FilterUtil.getKeyWithIndexesAndValues(listOfStartKeyByteArray));
-
- searchEndKey = FilterUtil
- .createIndexKeyFromResolvedFilterVal(endKey, segmentProperties.getDimensionKeyGenerator(),
- FilterUtil.getKeyWithIndexesAndValues(listOfEndKeyByteArray));
- listOfStartEndKeys.add(searchStartKey);
- listOfStartEndKeys.add(searchEndKey);
-
- }
-
- private static int compareFilterMembersBasedOnActualDataType(String filterMember1,
- String filterMember2, DataType dataType) {
- try {
- switch (dataType) {
- case SHORT:
- case INT:
- case LONG:
- case DOUBLE:
-
- if (CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(filterMember1)) {
- return 1;
- }
- Double d1 = Double.parseDouble(filterMember1);
- Double d2 = Double.parseDouble(filterMember2);
- return d1.compareTo(d2);
- case DECIMAL:
- if (CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(filterMember1)) {
- return 1;
- }
- java.math.BigDecimal val1 = new BigDecimal(filterMember1);
- java.math.BigDecimal val2 = new BigDecimal(filterMember2);
- return val1.compareTo(val2);
- case DATE:
- case TIMESTAMP:
- if (CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(filterMember1)) {
- return 1;
- }
- SimpleDateFormat parser = new SimpleDateFormat(CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
- Date date1 = null;
- Date date2 = null;
- date1 = parser.parse(filterMember1);
- date2 = parser.parse(filterMember2);
- return date1.compareTo(date2);
- case STRING:
- default:
- return filterMember1.compareTo(filterMember2);
- }
- } catch (Exception e) {
- return -1;
- }
- }
-
- private static void fillNullValuesStartIndexWithDefaultKeys(
- SortedMap<Integer, byte[]> setOfStartKeyByteArray, SegmentProperties segmentProperties) {
- List<CarbonDimension> allDimension = segmentProperties.getDimensions();
- for (CarbonDimension dimension : allDimension) {
- if (CarbonUtil.hasEncoding(dimension.getEncoder(), Encoding.DICTIONARY)) {
- continue;
- }
- if (null == setOfStartKeyByteArray.get(dimension.getOrdinal())) {
- setOfStartKeyByteArray.put(dimension.getOrdinal(), new byte[] { 0 });
- }
-
- }
- }
-
- private static void fillNullValuesEndIndexWithDefaultKeys(
- SortedMap<Integer, byte[]> setOfStartKeyByteArray, SegmentProperties segmentProperties) {
- List<CarbonDimension> allDimension = segmentProperties.getDimensions();
- for (CarbonDimension dimension : allDimension) {
- if (CarbonUtil.hasEncoding(dimension.getEncoder(), Encoding.DICTIONARY)) {
- continue;
- }
- if (null == setOfStartKeyByteArray.get(dimension.getOrdinal())) {
- setOfStartKeyByteArray.put(dimension.getOrdinal(), new byte[] { 127 });
- }
-
- }
- }
-
- private static void pruneStartAndEndKeys(SortedMap<Integer, byte[]> setOfStartKeyByteArray,
- List<byte[]> listOfStartKeyByteArray) {
- for (Map.Entry<Integer, byte[]> entry : setOfStartKeyByteArray.entrySet()) {
- listOfStartKeyByteArray.add(entry.getValue());
- }
- }
-
- private static void fillDefaultStartValue(SortedMap<Integer, byte[]> setOfStartKeyByteArray,
- SegmentProperties segmentProperties) {
- List<CarbonDimension> allDimension = segmentProperties.getDimensions();
- for (CarbonDimension dimension : allDimension) {
- if (CarbonUtil.hasEncoding(dimension.getEncoder(), Encoding.DICTIONARY)) {
- continue;
- }
- setOfStartKeyByteArray.put(dimension.getOrdinal(), new byte[] { 0 });
- }
-
- }
-
- private static void fillDefaultEndValue(SortedMap<Integer, byte[]> setOfEndKeyByteArray,
- SegmentProperties segmentProperties) {
- List<CarbonDimension> allDimension = segmentProperties.getDimensions();
- for (CarbonDimension dimension : allDimension) {
- if (CarbonUtil.hasEncoding(dimension.getEncoder(), Encoding.DICTIONARY)) {
- continue;
- }
- setOfEndKeyByteArray.put(dimension.getOrdinal(), new byte[] { 127 });
- }
- }
-
- private static void traverseResolverTreeAndPopulateStartAndEndKeys(
- FilterResolverIntf filterResolverTree, SegmentProperties segmentProperties, long[] startKeys,
- SortedMap<Integer, byte[]> setOfStartKeyByteArray, long[] endKeys,
- SortedMap<Integer, byte[]> setOfEndKeyByteArray, List<long[]> startKeyList,
- List<long[]> endKeyList) {
- if (null == filterResolverTree) {
- return;
- }
- traverseResolverTreeAndPopulateStartAndEndKeys(filterResolverTree.getLeft(),
- segmentProperties, startKeys, setOfStartKeyByteArray, endKeys, setOfEndKeyByteArray,
- startKeyList, endKeyList);
- filterResolverTree.getStartKey(startKeys, setOfStartKeyByteArray, startKeyList);
- filterResolverTree.getEndKey(segmentProperties, endKeys, setOfEndKeyByteArray,
- endKeyList);
-
- traverseResolverTreeAndPopulateStartAndEndKeys(filterResolverTree.getRight(),
- segmentProperties, startKeys, setOfStartKeyByteArray, endKeys, setOfEndKeyByteArray,
- startKeyList, endKeyList);
- }
-
- /**
- * Method will find whether the expression needs to be resolved, this can happen
- * if the expression is exclude and data type is null(mainly in IS NOT NULL filter scenario)
- *
- * @param rightExp
- * @param isIncludeFilter
- * @return
- */
- public static boolean isExpressionNeedsToResolved(Expression rightExp, boolean isIncludeFilter) {
- if (!isIncludeFilter && rightExp instanceof LiteralExpression && (
- DataType.NULL == ((LiteralExpression) rightExp)
- .getLiteralExpDataType())) {
- return true;
- }
- for (Expression child : rightExp.getChildren()) {
- if (isExpressionNeedsToResolved(child, isIncludeFilter)) {
- return true;
- }
- }
- return false;
- }
-
- /**
- * This method will print the error log.
- *
- * @param e
- */
- public static void logError(Throwable e, boolean invalidRowsPresent) {
- if (!invalidRowsPresent) {
- LOGGER.error(e, CarbonCommonConstants.FILTER_INVALID_MEMBER + e.getMessage());
- }
- }
-
- /**
- * This method will compare double values for its equality and also it will preserve
- * the -0.0 and 0.0 equality as per == ,also preserve NaN equality check as per
- * java.lang.Double.equals()
- *
- * @param d1 double value for equality check
- * @param d2 double value for equality check
- * @return boolean after comparing two double values.
- */
- public static boolean nanSafeEqualsDoubles(Double d1, Double d2) {
- if ((d1.doubleValue() == d2.doubleValue()) || (Double.isNaN(d1) && Double.isNaN(d2))) {
- return true;
- }
- return false;
-
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/GenericQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/GenericQueryType.java b/core/src/main/java/org/apache/carbondata/scan/filter/GenericQueryType.java
deleted file mode 100644
index f291a3a..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/GenericQueryType.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.filter;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.scan.processor.BlocksChunkHolder;
-
-public interface GenericQueryType {
-
- String getName();
-
- void setName(String name);
-
- String getParentname();
-
- void setParentname(String parentname);
-
- void addChildren(GenericQueryType children);
-
- int getColsCount();
-
- void parseBlocksAndReturnComplexColumnByteArray(DimensionColumnDataChunk[] dimensionDataChunks,
- int rowNumber, DataOutputStream dataOutputStream) throws IOException;
-
- void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder) throws IOException;
-
- Object getDataBasedOnDataTypeFromSurrogates(ByteBuffer surrogateData);
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/executer/AndFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/AndFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/AndFilterExecuterImpl.java
deleted file mode 100644
index ff4f8ce..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/AndFilterExecuterImpl.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.executer;
-
-import java.io.IOException;
-import java.util.BitSet;
-
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.processor.BlocksChunkHolder;
-
-public class AndFilterExecuterImpl implements FilterExecuter {
-
- private FilterExecuter leftExecuter;
- private FilterExecuter rightExecuter;
-
- public AndFilterExecuterImpl(FilterExecuter leftExecuter, FilterExecuter rightExecuter) {
- this.leftExecuter = leftExecuter;
- this.rightExecuter = rightExecuter;
- }
-
- @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
- throws FilterUnsupportedException, IOException {
- BitSet leftFilters = leftExecuter.applyFilter(blockChunkHolder);
- if (leftFilters.isEmpty()) {
- return leftFilters;
- }
- BitSet rightFilter = rightExecuter.applyFilter(blockChunkHolder);
- if (rightFilter.isEmpty()) {
- return rightFilter;
- }
- leftFilters.and(rightFilter);
- return leftFilters;
- }
-
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
- BitSet leftFilters = leftExecuter.isScanRequired(blockMaxValue, blockMinValue);
- if (leftFilters.isEmpty()) {
- return leftFilters;
- }
- BitSet rightFilter = rightExecuter.isScanRequired(blockMaxValue, blockMinValue);
- if (rightFilter.isEmpty()) {
- return rightFilter;
- }
- leftFilters.and(rightFilter);
- return leftFilters;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/executer/DimColumnExecuterFilterInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/DimColumnExecuterFilterInfo.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/DimColumnExecuterFilterInfo.java
deleted file mode 100644
index a54fb49..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/DimColumnExecuterFilterInfo.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.executer;
-
-public class DimColumnExecuterFilterInfo {
-
- byte[][] filterKeys;
-
- public void setFilterKeys(byte[][] filterKeys) {
- this.filterKeys = filterKeys;
- }
-
- public byte[][] getFilterKeys() {
- return filterKeys;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
deleted file mode 100644
index 2bb41e8..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.executer;
-
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.List;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
-import org.apache.carbondata.core.util.ByteUtil;
-import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.scan.executor.util.QueryUtil;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-
-/**
- * It checks if filter is required on given block and if required, it does
- * linear search on block data and set the bitset.
- */
-public class ExcludeColGroupFilterExecuterImpl extends ExcludeFilterExecuterImpl {
-
- /**
- * LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(ExcludeColGroupFilterExecuterImpl.class.getName());
-
- /**
- * @param dimColResolvedFilterInfo
- * @param segmentProperties
- */
- public ExcludeColGroupFilterExecuterImpl(DimColumnResolvedFilterInfo dimColResolvedFilterInfo,
- SegmentProperties segmentProperties) {
- super(dimColResolvedFilterInfo, segmentProperties);
- }
-
- /**
- * It fills BitSet with row index which matches filter key
- */
- protected BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
- int numerOfRows) {
- BitSet bitSet = new BitSet(numerOfRows);
- bitSet.flip(0, numerOfRows);
- try {
- KeyStructureInfo keyStructureInfo = getKeyStructureInfo();
- byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
- for (int i = 0; i < filterValues.length; i++) {
- byte[] filterVal = filterValues[i];
- for (int rowId = 0; rowId < numerOfRows; rowId++) {
- byte[] colData = new byte[keyStructureInfo.getMaskByteRanges().length];
- dimensionColumnDataChunk.fillChunkData(colData, 0, rowId, keyStructureInfo);
- if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterVal, colData) == 0) {
- bitSet.flip(rowId);
- }
- }
- }
-
- } catch (Exception e) {
- LOGGER.error(e);
- }
-
- return bitSet;
- }
-
- /**
- * It is required for extracting column data from columngroup chunk
- *
- * @return
- * @throws KeyGenException
- */
- private KeyStructureInfo getKeyStructureInfo() throws KeyGenException {
- int colGrpId = getColumnGroupId(dimColEvaluatorInfo.getColumnIndex());
- KeyGenerator keyGenerator = segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
- List<Integer> mdKeyOrdinal = new ArrayList<Integer>();
- mdKeyOrdinal.add(getMdkeyOrdinal(dimColEvaluatorInfo.getColumnIndex(), colGrpId));
- int[] maskByteRanges = QueryUtil.getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
- byte[] maxKey = QueryUtil.getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator);
- KeyStructureInfo restructureInfos = new KeyStructureInfo();
- restructureInfos.setKeyGenerator(keyGenerator);
- restructureInfos.setMaskByteRanges(maskByteRanges);
- restructureInfos.setMaxKey(maxKey);
- return restructureInfos;
- }
-
- /**
- * Check if scan is required on given block based on min and max value
- */
- public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) {
- BitSet bitSet = new BitSet(1);
- bitSet.flip(0, 1);
- return bitSet;
- }
-
- private int getMdkeyOrdinal(int ordinal, int colGrpId) {
- return segmentProperties.getColumnGroupMdKeyOrdinal(colGrpId, ordinal);
- }
-
- private int getColumnGroupId(int ordinal) {
- int[][] columnGroups = segmentProperties.getColumnGroups();
- int colGrpId = -1;
- for (int i = 0; i < columnGroups.length; i++) {
- if (columnGroups[i].length > 1) {
- colGrpId++;
- if (QueryUtil.searchInArray(columnGroups[i], ordinal)) {
- break;
- }
- }
- }
- return colGrpId;
- }
-
- public KeyGenerator getKeyGenerator(int colGrpId) {
- return segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeFilterExecuterImpl.java
deleted file mode 100644
index ddb887f..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/ExcludeFilterExecuterImpl.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.executer;
-
-import java.io.IOException;
-import java.util.BitSet;
-
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.scan.filter.FilterUtil;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.scan.processor.BlocksChunkHolder;
-
-public class ExcludeFilterExecuterImpl implements FilterExecuter {
-
- protected DimColumnResolvedFilterInfo dimColEvaluatorInfo;
- protected DimColumnExecuterFilterInfo dimColumnExecuterInfo;
- protected SegmentProperties segmentProperties;
-
- public ExcludeFilterExecuterImpl(DimColumnResolvedFilterInfo dimColEvaluatorInfo,
- SegmentProperties segmentProperties) {
- this.dimColEvaluatorInfo = dimColEvaluatorInfo;
- dimColumnExecuterInfo = new DimColumnExecuterFilterInfo();
- this.segmentProperties = segmentProperties;
- FilterUtil.prepareKeysFromSurrogates(dimColEvaluatorInfo.getFilterValues(), segmentProperties,
- dimColEvaluatorInfo.getDimension(), dimColumnExecuterInfo);
- }
-
- @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException {
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
- .get(dimColEvaluatorInfo.getColumnIndex());
- if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
- }
- return getFilteredIndexes(blockChunkHolder.getDimensionDataChunk()[blockIndex],
- blockChunkHolder.getDataBlock().nodeSize());
- }
-
- protected BitSet getFilteredIndexes(DimensionColumnDataChunk dimColumnDataChunk,
- int numerOfRows) {
- // For high cardinality dimensions.
- if (dimColumnDataChunk.isNoDicitionaryColumn()
- && dimColumnDataChunk instanceof VariableLengthDimensionDataChunk) {
- return setDirectKeyFilterIndexToBitSet((VariableLengthDimensionDataChunk) dimColumnDataChunk,
- numerOfRows);
- }
- if (dimColumnDataChunk.isExplicitSorted()
- && dimColumnDataChunk instanceof FixedLengthDimensionDataChunk) {
- return setFilterdIndexToBitSetWithColumnIndex(
- (FixedLengthDimensionDataChunk) dimColumnDataChunk, numerOfRows);
- }
- return setFilterdIndexToBitSet((FixedLengthDimensionDataChunk) dimColumnDataChunk, numerOfRows);
- }
-
- private BitSet setDirectKeyFilterIndexToBitSet(
- VariableLengthDimensionDataChunk dimColumnDataChunk, int numerOfRows) {
- BitSet bitSet = new BitSet(numerOfRows);
- bitSet.flip(0, numerOfRows);
- byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
- for (int i = 0; i < filterValues.length; i++) {
- byte[] filterVal = filterValues[i];
- if (dimColumnDataChunk.isExplicitSorted()) {
- for (int index = 0; index < numerOfRows; index++) {
- if (dimColumnDataChunk.compareTo(index, filterVal) == 0) {
- bitSet.flip(dimColumnDataChunk.getInvertedIndex(index));
- }
- }
- } else {
- for (int index = 0; index < numerOfRows; index++) {
- if (dimColumnDataChunk.compareTo(index, filterVal) == 0) {
- bitSet.flip(index);
- }
- }
- }
- }
- return bitSet;
-
- }
-
- private BitSet setFilterdIndexToBitSetWithColumnIndex(
- FixedLengthDimensionDataChunk dimColumnDataChunk, int numerOfRows) {
- int startKey = 0;
- int last = 0;
- int startIndex = 0;
- BitSet bitSet = new BitSet(numerOfRows);
- bitSet.flip(0, numerOfRows);
- byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
- for (int i = 0; i < filterValues.length; i++) {
- startKey = CarbonUtil
- .getFirstIndexUsingBinarySearch(dimColumnDataChunk, startIndex, numerOfRows - 1,
- filterValues[i], false);
- if (startKey < 0) {
- continue;
- }
- bitSet.flip(dimColumnDataChunk.getInvertedIndex(startKey));
- last = startKey;
- for (int j = startKey + 1; j < numerOfRows; j++) {
- if (dimColumnDataChunk.compareTo(j, filterValues[i]) == 0) {
- bitSet.flip(dimColumnDataChunk.getInvertedIndex(j));
- last++;
- } else {
- break;
- }
- }
- startIndex = last;
- if (startIndex >= numerOfRows) {
- break;
- }
- }
- return bitSet;
- }
-
- private BitSet setFilterdIndexToBitSet(FixedLengthDimensionDataChunk dimColumnDataChunk,
- int numerOfRows) {
- BitSet bitSet = new BitSet(numerOfRows);
- bitSet.flip(0, numerOfRows);
- byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
- for (int k = 0; k < filterValues.length; k++) {
- for (int j = 0; j < numerOfRows; j++) {
- if (dimColumnDataChunk.compareTo(j, filterValues[k]) == 0) {
- bitSet.flip(j);
- }
- }
- }
- return bitSet;
- }
-
- @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
- BitSet bitSet = new BitSet(1);
- bitSet.flip(0, 1);
- return bitSet;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/executer/FilterExecuter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/FilterExecuter.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/FilterExecuter.java
deleted file mode 100644
index 8d03093..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/FilterExecuter.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.executer;
-
-import java.io.IOException;
-import java.util.BitSet;
-
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.processor.BlocksChunkHolder;
-
-public interface FilterExecuter {
-
- /**
- * API will apply filter based on resolver instance
- *
- * @return
- * @throws FilterUnsupportedException
- */
- BitSet applyFilter(BlocksChunkHolder blocksChunkHolder)
- throws FilterUnsupportedException, IOException;
-
- /**
- * API will verify whether the block can be shortlisted based on block
- * max and min key.
- *
- * @param blockMaxValue, maximum value of the
- * @param blockMinValue
- * @return BitSet
- */
- BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue);
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
deleted file mode 100644
index f9e0bb2..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
+++ /dev/null
@@ -1,207 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.executer;
-
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.List;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
-import org.apache.carbondata.core.util.ByteUtil;
-import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.scan.executor.util.QueryUtil;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-
-/**
- * It checks if filter is required on given block and if required, it does
- * linear search on block data and set the bitset.
- */
-public class IncludeColGroupFilterExecuterImpl extends IncludeFilterExecuterImpl {
-
- /**
- * LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(IncludeColGroupFilterExecuterImpl.class.getName());
-
- /**
- * @param dimColResolvedFilterInfo
- * @param segmentProperties
- */
- public IncludeColGroupFilterExecuterImpl(DimColumnResolvedFilterInfo dimColResolvedFilterInfo,
- SegmentProperties segmentProperties) {
- super(dimColResolvedFilterInfo, segmentProperties);
- }
-
- /**
- * It fills BitSet with row index which matches filter key
- */
- protected BitSet getFilteredIndexes(DimensionColumnDataChunk dimensionColumnDataChunk,
- int numerOfRows) {
- BitSet bitSet = new BitSet(numerOfRows);
-
- try {
- KeyStructureInfo keyStructureInfo = getKeyStructureInfo();
- byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
- for (int i = 0; i < filterValues.length; i++) {
- byte[] filterVal = filterValues[i];
- for (int rowId = 0; rowId < numerOfRows; rowId++) {
- byte[] colData = new byte[keyStructureInfo.getMaskByteRanges().length];
- dimensionColumnDataChunk.fillChunkData(colData, 0, rowId, keyStructureInfo);
- if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterVal, colData) == 0) {
- bitSet.set(rowId);
- }
- }
- }
-
- } catch (Exception e) {
- LOGGER.error(e);
- }
-
- return bitSet;
- }
-
- /**
- * It is required for extracting column data from columngroup chunk
- *
- * @return
- * @throws KeyGenException
- */
- private KeyStructureInfo getKeyStructureInfo() throws KeyGenException {
- int colGrpId = getColumnGroupId(dimColumnEvaluatorInfo.getColumnIndex());
- KeyGenerator keyGenerator = segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
- List<Integer> mdKeyOrdinal = new ArrayList<Integer>();
- mdKeyOrdinal.add(getMdkeyOrdinal(dimColumnEvaluatorInfo.getColumnIndex(), colGrpId));
- int[] maskByteRanges = QueryUtil.getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
- byte[] maxKey = QueryUtil.getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator);
- KeyStructureInfo restructureInfos = new KeyStructureInfo();
- restructureInfos.setKeyGenerator(keyGenerator);
- restructureInfos.setMaskByteRanges(maskByteRanges);
- restructureInfos.setMaxKey(maxKey);
- return restructureInfos;
- }
-
- /**
- * Check if scan is required on given block based on min and max value
- */
- public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) {
- BitSet bitSet = new BitSet(1);
- byte[][] filterValues = dimColumnExecuterInfo.getFilterKeys();
- int columnIndex = dimColumnEvaluatorInfo.getColumnIndex();
- int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping().get(columnIndex);
- int[] cols = getAllColumns(columnIndex);
- byte[] maxValue = getMinMaxData(cols, blkMaxVal[blockIndex], columnIndex);
- byte[] minValue = getMinMaxData(cols, blkMinVal[blockIndex], columnIndex);
- boolean isScanRequired = false;
- for (int k = 0; k < filterValues.length; k++) {
- // filter value should be in range of max and min value i.e
- // max>filtervalue>min
- // so filter-max should be negative
- int maxCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], maxValue);
- // and filter-min should be positive
- int minCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], minValue);
-
- // if any filter value is in range than this block needs to be
- // scanned
- if (maxCompare <= 0 && minCompare >= 0) {
- isScanRequired = true;
- break;
- }
- }
- if (isScanRequired) {
- bitSet.set(0);
- }
- return bitSet;
- }
-
- /**
- * It extract min and max data for given column from stored min max value
- *
- * @param colGrpColumns
- * @param minMaxData
- * @param columnIndex
- * @return
- */
- private byte[] getMinMaxData(int[] colGrpColumns, byte[] minMaxData, int columnIndex) {
- int startIndex = 0;
- int endIndex = 0;
- if (null != colGrpColumns) {
- for (int i = 0; i < colGrpColumns.length; i++) {
- int colGrpId = getColumnGroupId(colGrpColumns[i]);
- int mdKeyOrdinal = getMdkeyOrdinal(colGrpColumns[i], colGrpId);
- int[] byteRange = getKeyGenerator(colGrpId).getKeyByteOffsets(mdKeyOrdinal);
- int colSize = 0;
- for (int j = byteRange[0]; j <= byteRange[1]; j++) {
- colSize++;
- }
- if (colGrpColumns[i] == columnIndex) {
- endIndex = startIndex + colSize;
- break;
- }
- startIndex += colSize;
- }
- }
- byte[] data = new byte[endIndex - startIndex];
- System.arraycopy(minMaxData, startIndex, data, 0, data.length);
- return data;
- }
-
- /**
- * It returns column groups which have provided column ordinal
- *
- * @param columnIndex
- * @return column group array
- */
- private int[] getAllColumns(int columnIndex) {
- int[][] colGroups = segmentProperties.getColumnGroups();
- for (int i = 0; i < colGroups.length; i++) {
- if (QueryUtil.searchInArray(colGroups[i], columnIndex)) {
- return colGroups[i];
- }
- }
- return null;
- }
-
- private int getMdkeyOrdinal(int ordinal, int colGrpId) {
- return segmentProperties.getColumnGroupMdKeyOrdinal(colGrpId, ordinal);
- }
-
- private int getColumnGroupId(int ordinal) {
- int[][] columnGroups = segmentProperties.getColumnGroups();
- int colGrpId = -1;
- for (int i = 0; i < columnGroups.length; i++) {
- if (columnGroups[i].length > 1) {
- colGrpId++;
- if (QueryUtil.searchInArray(columnGroups[i], ordinal)) {
- break;
- }
- }
- }
- return colGrpId;
- }
-
- public KeyGenerator getKeyGenerator(int colGrpId) {
- return segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
- }
-}
\ No newline at end of file
[20/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
new file mode 100644
index 0000000..7875f92
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/DetailQueryResultIterator.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.result.iterator;
+
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.core.scan.result.BatchResult;
+
+/**
+ * In case of detail query we cannot keep all the records in memory so for
+ * executing that query are returning a iterator over block and every time next
+ * call will come it will execute the block and return the result
+ */
+public class DetailQueryResultIterator extends AbstractDetailQueryResultIterator<BatchResult> {
+
+ private final Object lock = new Object();
+ private Future<BatchResult> future;
+
+ public DetailQueryResultIterator(List<BlockExecutionInfo> infos, QueryModel queryModel,
+ ExecutorService execService) {
+ super(infos, queryModel, execService);
+ }
+
+ @Override public BatchResult next() {
+ BatchResult result;
+ long startTime = System.currentTimeMillis();
+ try {
+ if (future == null) {
+ future = execute();
+ }
+ result = future.get();
+ nextBatch = false;
+ if (hasNext()) {
+ nextBatch = true;
+ future = execute();
+ } else {
+ fileReader.finish();
+ }
+ totalScanTime += System.currentTimeMillis() - startTime;
+ } catch (Exception ex) {
+ try {
+ fileReader.finish();
+ } finally {
+ throw new RuntimeException(ex);
+ }
+ }
+ return result;
+ }
+
+ private Future<BatchResult> execute() {
+ return execService.submit(new Callable<BatchResult>() {
+ @Override public BatchResult call() {
+ BatchResult batchResult = new BatchResult();
+ synchronized (lock) {
+ updateDataBlockIterator();
+ if (dataBlockIterator != null) {
+ batchResult.setRows(dataBlockIterator.next());
+ }
+ }
+ return batchResult;
+ }
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java
new file mode 100644
index 0000000..2f6a8b7
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/RawResultIterator.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.result.iterator;
+
+import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.scan.result.BatchResult;
+import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
+
+/**
+ * This is a wrapper iterator over the detail raw query iterator.
+ * This iterator will handle the processing of the raw rows.
+ * This will handle the batch results and will iterate on the batches and give single row.
+ */
+public class RawResultIterator extends CarbonIterator<Object[]> {
+
+ private final SegmentProperties sourceSegProperties;
+
+ private final SegmentProperties destinationSegProperties;
+ /**
+ * Iterator of the Batch raw result.
+ */
+ private CarbonIterator<BatchResult> detailRawQueryResultIterator;
+
+ /**
+ * Counter to maintain the row counter.
+ */
+ private int counter = 0;
+
+ private Object[] currentConveretedRawRow = null;
+
+ /**
+ * LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(RawResultIterator.class.getName());
+
+ /**
+ * batch of the result.
+ */
+ private BatchResult batch;
+
+ public RawResultIterator(CarbonIterator<BatchResult> detailRawQueryResultIterator,
+ SegmentProperties sourceSegProperties, SegmentProperties destinationSegProperties) {
+ this.detailRawQueryResultIterator = detailRawQueryResultIterator;
+ this.sourceSegProperties = sourceSegProperties;
+ this.destinationSegProperties = destinationSegProperties;
+ }
+
+ @Override public boolean hasNext() {
+
+ if (null == batch || checkIfBatchIsProcessedCompletely(batch)) {
+ if (detailRawQueryResultIterator.hasNext()) {
+ batch = null;
+ batch = detailRawQueryResultIterator.next();
+ counter = 0; // batch changed so reset the counter.
+ } else {
+ return false;
+ }
+ }
+
+ if (!checkIfBatchIsProcessedCompletely(batch)) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override public Object[] next() {
+ if (null == batch) { // for 1st time
+ batch = detailRawQueryResultIterator.next();
+ }
+ if (!checkIfBatchIsProcessedCompletely(batch)) {
+ try {
+ if(null != currentConveretedRawRow){
+ counter++;
+ Object[] currentConveretedRawRowTemp = this.currentConveretedRawRow;
+ currentConveretedRawRow = null;
+ return currentConveretedRawRowTemp;
+ }
+ return convertRow(batch.getRawRow(counter++));
+ } catch (KeyGenException e) {
+ LOGGER.error(e.getMessage());
+ return null;
+ }
+ } else { // completed one batch.
+ batch = null;
+ batch = detailRawQueryResultIterator.next();
+ counter = 0;
+ }
+ try {
+ if(null != currentConveretedRawRow){
+ counter++;
+ Object[] currentConveretedRawRowTemp = this.currentConveretedRawRow;
+ currentConveretedRawRow = null;
+ return currentConveretedRawRowTemp;
+ }
+
+ return convertRow(batch.getRawRow(counter++));
+ } catch (KeyGenException e) {
+ LOGGER.error(e.getMessage());
+ return null;
+ }
+
+ }
+
+ /**
+ * for fetching the row with out incrementing counter.
+ * @return
+ */
+ public Object[] fetchConverted() throws KeyGenException {
+ if(null != currentConveretedRawRow){
+ return currentConveretedRawRow;
+ }
+ if(hasNext())
+ {
+ Object[] rawRow = batch.getRawRow(counter);
+ currentConveretedRawRow = convertRow(rawRow);
+ return currentConveretedRawRow;
+ }
+ else
+ {
+ return null;
+ }
+ }
+
+ private Object[] convertRow(Object[] rawRow) throws KeyGenException {
+ byte[] dims = ((ByteArrayWrapper) rawRow[0]).getDictionaryKey();
+ long[] keyArray = sourceSegProperties.getDimensionKeyGenerator().getKeyArray(dims);
+ byte[] covertedBytes =
+ destinationSegProperties.getDimensionKeyGenerator().generateKey(keyArray);
+ ((ByteArrayWrapper) rawRow[0]).setDictionaryKey(covertedBytes);
+ return rawRow;
+ }
+
+ /**
+ * To check if the batch is processed completely
+ * @param batch
+ * @return
+ */
+ private boolean checkIfBatchIsProcessedCompletely(BatchResult batch){
+ if(counter < batch.getSize())
+ {
+ return false;
+ }
+ else{
+ return true;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorChunkRowIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorChunkRowIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorChunkRowIterator.java
new file mode 100644
index 0000000..b412655
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorChunkRowIterator.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.result.iterator;
+
+import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
+
+/**
+ * Iterator over row result
+ */
+public class VectorChunkRowIterator extends CarbonIterator<Object[]> {
+
+ /**
+ * iterator over chunk result
+ */
+ private AbstractDetailQueryResultIterator iterator;
+
+ /**
+ * currect chunk
+ */
+ private CarbonColumnarBatch columnarBatch;
+
+ private int batchSize;
+
+ private int currentIndex;
+
+ public VectorChunkRowIterator(AbstractDetailQueryResultIterator iterator,
+ CarbonColumnarBatch columnarBatch) {
+ this.iterator = iterator;
+ this.columnarBatch = columnarBatch;
+ if (iterator.hasNext()) {
+ iterator.processNextBatch(columnarBatch);
+ batchSize = columnarBatch.getActualSize();
+ currentIndex = 0;
+ }
+ }
+
+ /**
+ * Returns {@code true} if the iteration has more elements. (In other words,
+ * returns {@code true} if {@link #next} would return an element rather than
+ * throwing an exception.)
+ *
+ * @return {@code true} if the iteration has more elements
+ */
+ @Override public boolean hasNext() {
+ if (currentIndex < batchSize) {
+ return true;
+ } else {
+ while (iterator.hasNext()) {
+ columnarBatch.reset();
+ iterator.processNextBatch(columnarBatch);
+ batchSize = columnarBatch.getActualSize();
+ currentIndex = 0;
+ if (currentIndex < batchSize) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Returns the next element in the iteration.
+ *
+ * @return the next element in the iteration
+ */
+ @Override public Object[] next() {
+ Object[] row = new Object[columnarBatch.columnVectors.length];
+ for (int i = 0; i < row.length; i++) {
+ row[i] = columnarBatch.columnVectors[i].getData(currentIndex);
+ }
+ currentIndex++;
+ return row;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java
new file mode 100644
index 0000000..00116c5
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/VectorDetailQueryResultIterator.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.result.iterator;
+
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
+
+/**
+ * It reads the data vector batch format
+ */
+public class VectorDetailQueryResultIterator extends AbstractDetailQueryResultIterator<Object> {
+
+ private final Object lock = new Object();
+
+ public VectorDetailQueryResultIterator(List<BlockExecutionInfo> infos, QueryModel queryModel,
+ ExecutorService execService) {
+ super(infos, queryModel, execService);
+ }
+
+ @Override public Object next() {
+ throw new UnsupportedOperationException("call processNextBatch instaed");
+ }
+
+ public void processNextBatch(CarbonColumnarBatch columnarBatch) {
+ synchronized (lock) {
+ updateDataBlockIterator();
+ if (dataBlockIterator != null) {
+ dataBlockIterator.processNextBatch(columnarBatch);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java
new file mode 100644
index 0000000..7d29b0f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnVector.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.scan.result.vector;
+
+import org.apache.spark.sql.types.Decimal;
+
+public interface CarbonColumnVector {
+
+ void putShort(int rowId, short value);
+
+ void putInt(int rowId, int value);
+
+ void putLong(int rowId, long value);
+
+ void putDecimal(int rowId, Decimal value, int precision);
+
+ void putDouble(int rowId, double value);
+
+ void putBytes(int rowId, byte[] value);
+
+ void putBytes(int rowId, int offset, int length, byte[] value);
+
+ void putNull(int rowId);
+
+ boolean isNull(int rowId);
+
+ void putObject(int rowId, Object obj);
+
+ Object getData(int rowId);
+
+ void reset();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnarBatch.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnarBatch.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnarBatch.java
new file mode 100644
index 0000000..faeffde
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/CarbonColumnarBatch.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.result.vector;
+
+public class CarbonColumnarBatch {
+
+ public CarbonColumnVector[] columnVectors;
+
+ private int batchSize;
+
+ private int actualSize;
+
+ private int rowCounter;
+
+ public CarbonColumnarBatch(CarbonColumnVector[] columnVectors, int batchSize) {
+ this.columnVectors = columnVectors;
+ this.batchSize = batchSize;
+ }
+
+ public int getBatchSize() {
+ return batchSize;
+ }
+
+ public int getActualSize() {
+ return actualSize;
+ }
+
+ public void setActualSize(int actualSize) {
+ this.actualSize = actualSize;
+ }
+
+ public void reset() {
+ actualSize = 0;
+ rowCounter = 0;
+ for (int i = 0; i < columnVectors.length; i++) {
+ columnVectors[i].reset();
+ }
+ }
+
+ public int getRowCounter() {
+ return rowCounter;
+ }
+
+ public void setRowCounter(int rowCounter) {
+ this.rowCounter = rowCounter;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/result/vector/ColumnVectorInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/ColumnVectorInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/ColumnVectorInfo.java
new file mode 100644
index 0000000..852abe9
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/ColumnVectorInfo.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.result.vector;
+
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
+import org.apache.carbondata.core.scan.filter.GenericQueryType;
+import org.apache.carbondata.core.scan.model.QueryDimension;
+import org.apache.carbondata.core.scan.model.QueryMeasure;
+
+public class ColumnVectorInfo implements Comparable<ColumnVectorInfo> {
+ public int offset;
+ public int size;
+ public CarbonColumnVector vector;
+ public int vectorOffset;
+ public QueryDimension dimension;
+ public QueryMeasure measure;
+ public int ordinal;
+ public DirectDictionaryGenerator directDictionaryGenerator;
+ public MeasureDataVectorProcessor.MeasureVectorFiller measureVectorFiller;
+ public GenericQueryType genericQueryType;
+
+ @Override public int compareTo(ColumnVectorInfo o) {
+ return ordinal - o.ordinal;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java
new file mode 100644
index 0000000..3c25efc
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/MeasureDataVectorProcessor.java
@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.result.vector;
+
+import java.math.BigDecimal;
+import java.util.BitSet;
+
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
+import org.apache.carbondata.core.metadata.DataType;
+
+import org.apache.spark.sql.types.Decimal;
+
+public class MeasureDataVectorProcessor {
+
+ public interface MeasureVectorFiller {
+
+ void fillMeasureVector(MeasureColumnDataChunk dataChunk, ColumnVectorInfo info);
+
+ void fillMeasureVectorForFilter(int[] rowMapping, MeasureColumnDataChunk dataChunk,
+ ColumnVectorInfo info);
+ }
+
+ public static class IntegralMeasureVectorFiller implements MeasureVectorFiller {
+
+ @Override
+ public void fillMeasureVector(MeasureColumnDataChunk dataChunk, ColumnVectorInfo info) {
+ int offset = info.offset;
+ int len = offset + info.size;
+ int vectorOffset = info.vectorOffset;
+ CarbonColumnVector vector = info.vector;
+ BitSet nullBitSet = dataChunk.getNullValueIndexHolder().getBitSet();
+ for (int i = offset; i < len; i++) {
+ if (nullBitSet.get(i)) {
+ vector.putNull(vectorOffset);
+ } else {
+ vector.putInt(vectorOffset,
+ (int)dataChunk.getMeasureDataHolder().getReadableLongValueByIndex(i));
+ }
+ vectorOffset++;
+ }
+ }
+
+ @Override
+ public void fillMeasureVectorForFilter(int[] rowMapping, MeasureColumnDataChunk dataChunk,
+ ColumnVectorInfo info) {
+ int offset = info.offset;
+ int len = offset + info.size;
+ int vectorOffset = info.vectorOffset;
+ CarbonColumnVector vector = info.vector;
+ BitSet nullBitSet = dataChunk.getNullValueIndexHolder().getBitSet();
+ for (int i = offset; i < len; i++) {
+ int currentRow = rowMapping[i];
+ if (nullBitSet.get(currentRow)) {
+ vector.putNull(vectorOffset);
+ } else {
+ vector.putInt(vectorOffset,
+ (int)dataChunk.getMeasureDataHolder().getReadableLongValueByIndex(currentRow));
+ }
+ vectorOffset++;
+ }
+ }
+ }
+
+ public static class ShortMeasureVectorFiller implements MeasureVectorFiller {
+
+ @Override
+ public void fillMeasureVector(MeasureColumnDataChunk dataChunk, ColumnVectorInfo info) {
+ int offset = info.offset;
+ int len = offset + info.size;
+ int vectorOffset = info.vectorOffset;
+ CarbonColumnVector vector = info.vector;
+ BitSet nullBitSet = dataChunk.getNullValueIndexHolder().getBitSet();
+ for (int i = offset; i < len; i++) {
+ if (nullBitSet.get(i)) {
+ vector.putNull(vectorOffset);
+ } else {
+ vector.putShort(vectorOffset,
+ (short) dataChunk.getMeasureDataHolder().getReadableLongValueByIndex(i));
+ }
+ vectorOffset++;
+ }
+ }
+
+ @Override
+ public void fillMeasureVectorForFilter(int[] rowMapping, MeasureColumnDataChunk dataChunk,
+ ColumnVectorInfo info) {
+ int offset = info.offset;
+ int len = offset + info.size;
+ int vectorOffset = info.vectorOffset;
+ CarbonColumnVector vector = info.vector;
+ BitSet nullBitSet = dataChunk.getNullValueIndexHolder().getBitSet();
+ for (int i = offset; i < len; i++) {
+ int currentRow = rowMapping[i];
+ if (nullBitSet.get(currentRow)) {
+ vector.putNull(vectorOffset);
+ } else {
+ vector.putShort(vectorOffset,
+ (short) dataChunk.getMeasureDataHolder().getReadableLongValueByIndex(currentRow));
+ }
+ vectorOffset++;
+ }
+ }
+ }
+
+ public static class LongMeasureVectorFiller implements MeasureVectorFiller {
+
+ @Override
+ public void fillMeasureVector(MeasureColumnDataChunk dataChunk, ColumnVectorInfo info) {
+ int offset = info.offset;
+ int len = offset + info.size;
+ int vectorOffset = info.vectorOffset;
+ CarbonColumnVector vector = info.vector;
+ BitSet nullBitSet = dataChunk.getNullValueIndexHolder().getBitSet();
+ for (int i = offset; i < len; i++) {
+ if (nullBitSet.get(i)) {
+ vector.putNull(vectorOffset);
+ } else {
+ vector.putLong(vectorOffset,
+ dataChunk.getMeasureDataHolder().getReadableLongValueByIndex(i));
+ }
+ vectorOffset++;
+ }
+ }
+
+ @Override
+ public void fillMeasureVectorForFilter(int[] rowMapping, MeasureColumnDataChunk dataChunk,
+ ColumnVectorInfo info) {
+ int offset = info.offset;
+ int len = offset + info.size;
+ int vectorOffset = info.vectorOffset;
+ CarbonColumnVector vector = info.vector;
+ BitSet nullBitSet = dataChunk.getNullValueIndexHolder().getBitSet();
+ for (int i = offset; i < len; i++) {
+ int currentRow = rowMapping[i];
+ if (nullBitSet.get(currentRow)) {
+ vector.putNull(vectorOffset);
+ } else {
+ vector.putLong(vectorOffset,
+ dataChunk.getMeasureDataHolder().getReadableLongValueByIndex(currentRow));
+ }
+ vectorOffset++;
+ }
+ }
+ }
+
+ public static class DecimalMeasureVectorFiller implements MeasureVectorFiller {
+
+ @Override
+ public void fillMeasureVector(MeasureColumnDataChunk dataChunk, ColumnVectorInfo info) {
+ int offset = info.offset;
+ int len = offset + info.size;
+ int vectorOffset = info.vectorOffset;
+ CarbonColumnVector vector = info.vector;
+ int precision = info.measure.getMeasure().getPrecision();
+ BitSet nullBitSet = dataChunk.getNullValueIndexHolder().getBitSet();
+ for (int i = offset; i < len; i++) {
+ if (nullBitSet.get(i)) {
+ vector.putNull(vectorOffset);
+ } else {
+ BigDecimal decimal =
+ dataChunk.getMeasureDataHolder().getReadableBigDecimalValueByIndex(i);
+ Decimal toDecimal = org.apache.spark.sql.types.Decimal.apply(decimal);
+ vector.putDecimal(vectorOffset, toDecimal, precision);
+ }
+ vectorOffset++;
+ }
+ }
+
+ @Override
+ public void fillMeasureVectorForFilter(int[] rowMapping, MeasureColumnDataChunk dataChunk,
+ ColumnVectorInfo info) {
+ int offset = info.offset;
+ int len = offset + info.size;
+ int vectorOffset = info.vectorOffset;
+ CarbonColumnVector vector = info.vector;
+ int precision = info.measure.getMeasure().getPrecision();
+ BitSet nullBitSet = dataChunk.getNullValueIndexHolder().getBitSet();
+ for (int i = offset; i < len; i++) {
+ int currentRow = rowMapping[i];
+ if (nullBitSet.get(currentRow)) {
+ vector.putNull(vectorOffset);
+ } else {
+ BigDecimal decimal =
+ dataChunk.getMeasureDataHolder().getReadableBigDecimalValueByIndex(currentRow);
+ Decimal toDecimal = org.apache.spark.sql.types.Decimal.apply(decimal);
+ vector.putDecimal(vectorOffset, toDecimal, precision);
+ }
+ vectorOffset++;
+ }
+ }
+ }
+
+ public static class DefaultMeasureVectorFiller implements MeasureVectorFiller {
+
+ @Override
+ public void fillMeasureVector(MeasureColumnDataChunk dataChunk, ColumnVectorInfo info) {
+ int offset = info.offset;
+ int len = offset + info.size;
+ int vectorOffset = info.vectorOffset;
+ CarbonColumnVector vector = info.vector;
+ BitSet nullBitSet = dataChunk.getNullValueIndexHolder().getBitSet();
+ for (int i = offset; i < len; i++) {
+ if (nullBitSet.get(i)) {
+ vector.putNull(vectorOffset);
+ } else {
+ vector.putDouble(vectorOffset,
+ dataChunk.getMeasureDataHolder().getReadableDoubleValueByIndex(i));
+ }
+ vectorOffset++;
+ }
+ }
+
+ @Override
+ public void fillMeasureVectorForFilter(int[] rowMapping, MeasureColumnDataChunk dataChunk,
+ ColumnVectorInfo info) {
+ int offset = info.offset;
+ int len = offset + info.size;
+ int vectorOffset = info.vectorOffset;
+ CarbonColumnVector vector = info.vector;
+ BitSet nullBitSet = dataChunk.getNullValueIndexHolder().getBitSet();
+ for (int i = offset; i < len; i++) {
+ int currentRow = rowMapping[i];
+ if (nullBitSet.get(currentRow)) {
+ vector.putNull(vectorOffset);
+ } else {
+ vector.putDouble(vectorOffset,
+ dataChunk.getMeasureDataHolder().getReadableDoubleValueByIndex(currentRow));
+ }
+ vectorOffset++;
+ }
+ }
+ }
+
+ public static class MeasureVectorFillerFactory {
+
+ public static MeasureVectorFiller getMeasureVectorFiller(DataType dataType) {
+ switch (dataType) {
+ case SHORT:
+ return new ShortMeasureVectorFiller();
+ case INT:
+ return new IntegralMeasureVectorFiller();
+ case LONG:
+ return new LongMeasureVectorFiller();
+ case DECIMAL:
+ return new DecimalMeasureVectorFiller();
+ default:
+ return new DefaultMeasureVectorFiller();
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/CarbonColumnVectorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/CarbonColumnVectorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/CarbonColumnVectorImpl.java
new file mode 100644
index 0000000..7ae1a41
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/vector/impl/CarbonColumnVectorImpl.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.result.vector.impl;
+
+import java.util.Arrays;
+import java.util.BitSet;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
+
+import org.apache.spark.sql.types.Decimal;
+import org.apache.spark.unsafe.types.UTF8String;
+
+public class CarbonColumnVectorImpl implements CarbonColumnVector {
+
+ private Object[] data;
+
+ private int[] ints;
+
+ private long[] longs;
+
+ private Decimal[] decimals;
+
+ private byte[][] bytes;
+
+ private double[] doubles;
+
+ private BitSet nullBytes;
+
+ private DataType dataType;
+
+ public CarbonColumnVectorImpl(int batchSize, DataType dataType) {
+ nullBytes = new BitSet(batchSize);
+ this.dataType = dataType;
+ switch (dataType) {
+ case INT:
+ ints = new int[batchSize];
+ break;
+ case LONG:
+ longs = new long[batchSize];
+ break;
+ case DOUBLE:
+ doubles = new double[batchSize];
+ break;
+ case STRING:
+ bytes = new byte[batchSize][];
+ break;
+ case DECIMAL:
+ decimals = new Decimal[batchSize];
+ break;
+ default:
+ data = new Object[batchSize];
+ }
+ }
+
+ @Override public void putShort(int rowId, short value) {
+
+ }
+
+ @Override public void putInt(int rowId, int value) {
+ ints[rowId] = value;
+ }
+
+ @Override public void putLong(int rowId, long value) {
+ longs[rowId] = value;
+ }
+
+ @Override public void putDecimal(int rowId, Decimal value, int precision) {
+ decimals[rowId] = value;
+ }
+
+ @Override public void putDouble(int rowId, double value) {
+ doubles[rowId] = value;
+ }
+
+ @Override public void putBytes(int rowId, byte[] value) {
+ bytes[rowId] = value;
+ }
+
+ @Override public void putBytes(int rowId, int offset, int length, byte[] value) {
+
+ }
+
+ @Override public void putNull(int rowId) {
+ nullBytes.set(rowId);
+ }
+
+ @Override public boolean isNull(int rowId) {
+ return nullBytes.get(rowId);
+ }
+
+ @Override public void putObject(int rowId, Object obj) {
+ data[rowId] = obj;
+ }
+
+ @Override public Object getData(int rowId) {
+ if (nullBytes.get(rowId)) {
+ return null;
+ }
+ switch (dataType) {
+ case INT:
+ return ints[rowId];
+ case LONG:
+ return longs[rowId];
+ case DOUBLE:
+ return doubles[rowId];
+ case STRING:
+ return UTF8String.fromBytes(bytes[rowId]);
+ case DECIMAL:
+ return decimals[rowId];
+ default:
+ return data[rowId];
+ }
+ }
+
+ @Override public void reset() {
+ nullBytes.clear();
+ switch (dataType) {
+ case INT:
+ Arrays.fill(ints, 0);
+ break;
+ case LONG:
+ Arrays.fill(longs, 0);
+ break;
+ case DOUBLE:
+ Arrays.fill(doubles, 0);
+ break;
+ case STRING:
+ Arrays.fill(bytes, null);
+ break;
+ case DECIMAL:
+ Arrays.fill(decimals, null);
+ break;
+ default:
+ Arrays.fill(data, null);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
new file mode 100644
index 0000000..32b3ac2
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.scanner;
+
+import java.io.IOException;
+
+import org.apache.carbondata.core.update.data.BlockletDeleteDeltaCacheLoader;
+import org.apache.carbondata.core.update.data.DeleteDeltaCacheLoaderIntf;
+import org.apache.carbondata.core.stats.QueryStatistic;
+import org.apache.carbondata.core.stats.QueryStatisticsConstants;
+import org.apache.carbondata.core.stats.QueryStatisticsModel;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+
+/**
+ * Blocklet scanner class to process the block
+ */
+public abstract class AbstractBlockletScanner implements BlockletScanner {
+
+ /**
+ * scanner result
+ */
+ protected AbstractScannedResult scannedResult;
+
+ /**
+ * block execution info
+ */
+ protected BlockExecutionInfo blockExecutionInfo;
+
+ public QueryStatisticsModel queryStatisticsModel;
+
+ public AbstractBlockletScanner(BlockExecutionInfo tableBlockExecutionInfos) {
+ this.blockExecutionInfo = tableBlockExecutionInfos;
+ }
+
+ @Override public AbstractScannedResult scanBlocklet(BlocksChunkHolder blocksChunkHolder)
+ throws IOException, FilterUnsupportedException {
+ fillKeyValue(blocksChunkHolder);
+ return scannedResult;
+ }
+
+ protected void fillKeyValue(BlocksChunkHolder blocksChunkHolder) throws IOException {
+
+ QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
+ totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM,
+ totalBlockletStatistic.getCount() + 1);
+ queryStatisticsModel.getRecorder().recordStatistics(totalBlockletStatistic);
+ QueryStatistic validScannedBlockletStatistic = queryStatisticsModel
+ .getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM);
+ validScannedBlockletStatistic
+ .addCountStatistic(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM,
+ validScannedBlockletStatistic.getCount() + 1);
+ queryStatisticsModel.getRecorder().recordStatistics(validScannedBlockletStatistic);
+ scannedResult.reset();
+ scannedResult.setNumberOfRows(blocksChunkHolder.getDataBlock().nodeSize());
+ scannedResult.setBlockletId(
+ blockExecutionInfo.getBlockId() + CarbonCommonConstants.FILE_SEPARATOR
+ + blocksChunkHolder.getDataBlock().nodeNumber());
+ scannedResult.setDimensionChunks(blocksChunkHolder.getDataBlock()
+ .getDimensionChunks(blocksChunkHolder.getFileReader(),
+ blockExecutionInfo.getAllSelectedDimensionBlocksIndexes()));
+ scannedResult.setMeasureChunks(blocksChunkHolder.getDataBlock()
+ .getMeasureChunks(blocksChunkHolder.getFileReader(),
+ blockExecutionInfo.getAllSelectedMeasureBlocksIndexes()));
+ // loading delete data cache in blockexecutioninfo instance
+ DeleteDeltaCacheLoaderIntf deleteCacheLoader =
+ new BlockletDeleteDeltaCacheLoader(scannedResult.getBlockletId(),
+ blocksChunkHolder.getDataBlock(), blockExecutionInfo.getAbsoluteTableIdentifier());
+ deleteCacheLoader.loadDeleteDeltaFileDataToCache();
+ scannedResult
+ .setBlockletDeleteDeltaCache(blocksChunkHolder.getDataBlock().getDeleteDeltaDataCache());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/scanner/BlockletScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/BlockletScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/BlockletScanner.java
new file mode 100644
index 0000000..9484318
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/BlockletScanner.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.scanner;
+
+import java.io.IOException;
+
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+
+/**
+ * Interface for processing the block
+ * Processing can be filter based processing or non filter based processing
+ */
+public interface BlockletScanner {
+
+ /**
+ * Below method will used to process the block data and get the scanned result
+ *
+ * @param blocksChunkHolder block chunk which holds the block data
+ * @return scannerResult
+ * result after processing
+ */
+ AbstractScannedResult scanBlocklet(BlocksChunkHolder blocksChunkHolder)
+ throws IOException, FilterUnsupportedException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
new file mode 100644
index 0000000..48c9af2
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.scanner.impl;
+
+import java.io.IOException;
+import java.util.BitSet;
+
+import org.apache.carbondata.core.update.data.BlockletDeleteDeltaCacheLoader;
+import org.apache.carbondata.core.update.data.DeleteDeltaCacheLoaderIntf;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
+import org.apache.carbondata.core.stats.QueryStatistic;
+import org.apache.carbondata.core.stats.QueryStatisticsConstants;
+import org.apache.carbondata.core.stats.QueryStatisticsModel;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.result.impl.FilterQueryScannedResult;
+import org.apache.carbondata.core.scan.scanner.AbstractBlockletScanner;
+
+/**
+ * Below class will be used for filter query processing
+ * this class will be first apply the filter then it will read the block if
+ * required and return the scanned result
+ */
+public class FilterScanner extends AbstractBlockletScanner {
+
+ /**
+ * filter tree
+ */
+ private FilterExecuter filterExecuter;
+ /**
+ * this will be used to apply min max
+ * this will be useful for dimension column which is on the right side
+ * as node finder will always give tentative blocks, if column data stored individually
+ * and data is in sorted order then we can check whether filter is in the range of min max or not
+ * if it present then only we can apply filter on complete data.
+ * this will be very useful in case of sparse data when rows are
+ * repeating.
+ */
+ private boolean isMinMaxEnabled;
+
+ private QueryStatisticsModel queryStatisticsModel;
+
+ public FilterScanner(BlockExecutionInfo blockExecutionInfo,
+ QueryStatisticsModel queryStatisticsModel) {
+ super(blockExecutionInfo);
+ scannedResult = new FilterQueryScannedResult(blockExecutionInfo);
+ // to check whether min max is enabled or not
+ String minMaxEnableValue = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.CARBON_QUERY_MIN_MAX_ENABLED,
+ CarbonCommonConstants.MIN_MAX_DEFAULT_VALUE);
+ if (null != minMaxEnableValue) {
+ isMinMaxEnabled = Boolean.parseBoolean(minMaxEnableValue);
+ }
+ // get the filter tree
+ this.filterExecuter = blockExecutionInfo.getFilterExecuterTree();
+ this.queryStatisticsModel = queryStatisticsModel;
+ }
+
+ /**
+ * Below method will be used to process the block
+ *
+ * @param blocksChunkHolder block chunk holder which holds the data
+ * @throws FilterUnsupportedException
+ */
+ @Override public AbstractScannedResult scanBlocklet(BlocksChunkHolder blocksChunkHolder)
+ throws IOException, FilterUnsupportedException {
+ fillScannedResult(blocksChunkHolder);
+ return scannedResult;
+ }
+
+ /**
+ * This method will process the data in below order
+ * 1. first apply min max on the filter tree and check whether any of the filter
+ * is fall on the range of min max, if not then return empty result
+ * 2. If filter falls on min max range then apply filter on actual
+ * data and get the filtered row index
+ * 3. if row index is empty then return the empty result
+ * 4. if row indexes is not empty then read only those blocks(measure or dimension)
+ * which was present in the query but not present in the filter, as while applying filter
+ * some of the blocks where already read and present in chunk holder so not need to
+ * read those blocks again, this is to avoid reading of same blocks which was already read
+ * 5. Set the blocks and filter indexes to result
+ *
+ * @param blocksChunkHolder
+ * @throws FilterUnsupportedException
+ */
+ private void fillScannedResult(BlocksChunkHolder blocksChunkHolder)
+ throws FilterUnsupportedException, IOException {
+ scannedResult.reset();
+ scannedResult.setBlockletId(
+ blockExecutionInfo.getBlockId() + CarbonCommonConstants.FILE_SEPARATOR + blocksChunkHolder
+ .getDataBlock().nodeNumber());
+ // apply min max
+ if (isMinMaxEnabled) {
+ BitSet bitSet = this.filterExecuter
+ .isScanRequired(blocksChunkHolder.getDataBlock().getColumnsMaxValue(),
+ blocksChunkHolder.getDataBlock().getColumnsMinValue());
+ if (bitSet.isEmpty()) {
+ scannedResult.setNumberOfRows(0);
+ scannedResult.setIndexes(new int[0]);
+ CarbonUtil.freeMemory(blocksChunkHolder.getDimensionDataChunk(),
+ blocksChunkHolder.getMeasureDataChunk());
+ return;
+ }
+ }
+ // apply filter on actual data
+ BitSet bitSet = this.filterExecuter.applyFilter(blocksChunkHolder);
+ // if indexes is empty then return with empty result
+ if (bitSet.isEmpty()) {
+ scannedResult.setNumberOfRows(0);
+ scannedResult.setIndexes(new int[0]);
+ return;
+ }
+ // valid scanned blocklet
+ QueryStatistic validScannedBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .get(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM);
+ validScannedBlockletStatistic
+ .addCountStatistic(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM,
+ validScannedBlockletStatistic.getCount() + 1);
+ queryStatisticsModel.getRecorder().recordStatistics(validScannedBlockletStatistic);
+ // get the row indexes from bot set
+ int[] indexes = new int[bitSet.cardinality()];
+ int index = 0;
+ for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) {
+ indexes[index++] = i;
+ }
+ // loading delete data cache in blockexecutioninfo instance
+ DeleteDeltaCacheLoaderIntf deleteCacheLoader =
+ new BlockletDeleteDeltaCacheLoader(scannedResult.getBlockletId(),
+ blocksChunkHolder.getDataBlock(), blockExecutionInfo.getAbsoluteTableIdentifier());
+ deleteCacheLoader.loadDeleteDeltaFileDataToCache();
+ scannedResult
+ .setBlockletDeleteDeltaCache(blocksChunkHolder.getDataBlock().getDeleteDeltaDataCache());
+ FileHolder fileReader = blocksChunkHolder.getFileReader();
+ int[][] allSelectedDimensionBlocksIndexes =
+ blockExecutionInfo.getAllSelectedDimensionBlocksIndexes();
+ DimensionColumnDataChunk[] projectionListDimensionChunk = blocksChunkHolder.getDataBlock()
+ .getDimensionChunks(fileReader, allSelectedDimensionBlocksIndexes);
+
+ DimensionColumnDataChunk[] dimensionColumnDataChunk =
+ new DimensionColumnDataChunk[blockExecutionInfo.getTotalNumberDimensionBlock()];
+ // read dimension chunk blocks from file which is not present
+ for (int i = 0; i < dimensionColumnDataChunk.length; i++) {
+ if (null != blocksChunkHolder.getDimensionDataChunk()[i]) {
+ dimensionColumnDataChunk[i] = blocksChunkHolder.getDimensionDataChunk()[i];
+ }
+ }
+ for (int i = 0; i < allSelectedDimensionBlocksIndexes.length; i++) {
+ System.arraycopy(projectionListDimensionChunk, allSelectedDimensionBlocksIndexes[i][0],
+ dimensionColumnDataChunk, allSelectedDimensionBlocksIndexes[i][0],
+ allSelectedDimensionBlocksIndexes[i][1] + 1 - allSelectedDimensionBlocksIndexes[i][0]);
+ }
+ MeasureColumnDataChunk[] measureColumnDataChunk =
+ new MeasureColumnDataChunk[blockExecutionInfo.getTotalNumberOfMeasureBlock()];
+ int[][] allSelectedMeasureBlocksIndexes =
+ blockExecutionInfo.getAllSelectedMeasureBlocksIndexes();
+ MeasureColumnDataChunk[] projectionListMeasureChunk = blocksChunkHolder.getDataBlock()
+ .getMeasureChunks(fileReader, allSelectedMeasureBlocksIndexes);
+ // read the measure chunk blocks which is not present
+ for (int i = 0; i < measureColumnDataChunk.length; i++) {
+ if (null != blocksChunkHolder.getMeasureDataChunk()[i]) {
+ measureColumnDataChunk[i] = blocksChunkHolder.getMeasureDataChunk()[i];
+ }
+ }
+ for (int i = 0; i < allSelectedMeasureBlocksIndexes.length; i++) {
+ System.arraycopy(projectionListMeasureChunk, allSelectedMeasureBlocksIndexes[i][0],
+ measureColumnDataChunk, allSelectedMeasureBlocksIndexes[i][0],
+ allSelectedMeasureBlocksIndexes[i][1] + 1 - allSelectedMeasureBlocksIndexes[i][0]);
+ }
+ scannedResult.setDimensionChunks(dimensionColumnDataChunk);
+ scannedResult.setIndexes(indexes);
+ scannedResult.setMeasureChunks(measureColumnDataChunk);
+ scannedResult.setNumberOfRows(indexes.length);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/NonFilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/NonFilterScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/NonFilterScanner.java
new file mode 100644
index 0000000..f07c2be
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/NonFilterScanner.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.scanner.impl;
+
+import org.apache.carbondata.core.stats.QueryStatisticsModel;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.result.impl.NonFilterQueryScannedResult;
+import org.apache.carbondata.core.scan.scanner.AbstractBlockletScanner;
+
+/**
+ * Non filter processor which will be used for non filter query
+ * In case of non filter query we just need to read all the blocks requested in the
+ * query and pass it to scanned result
+ */
+public class NonFilterScanner extends AbstractBlockletScanner {
+
+ public NonFilterScanner(BlockExecutionInfo blockExecutionInfo,
+ QueryStatisticsModel queryStatisticsModel) {
+ super(blockExecutionInfo);
+ // as its a non filter query creating a non filter query scanned result object
+ scannedResult = new NonFilterQueryScannedResult(blockExecutionInfo);
+ super.queryStatisticsModel = queryStatisticsModel;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java b/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java
new file mode 100644
index 0000000..68eb946
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapper.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.wrappers;
+
+import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
+
+/**
+ * This class will store the dimension column data when query is executed
+ * This can be used as a key for aggregation
+ */
+public class ByteArrayWrapper implements Comparable<ByteArrayWrapper> {
+
+ /**
+ * to store key which is generated using
+ * key generator
+ */
+ protected byte[] dictionaryKey;
+
+ /**
+ * to store no dictionary column data
+ */
+ protected byte[][] complexTypesKeys;
+
+ /**
+ * to store no dictionary column data
+ */
+ protected byte[][] noDictionaryKeys;
+
+ /**
+ * contains value of implicit columns in byte array format
+ */
+ protected byte[] implicitColumnByteArray;
+
+ public ByteArrayWrapper() {
+ }
+
+ /**
+ * @return the dictionaryKey
+ */
+ public byte[] getDictionaryKey() {
+ return dictionaryKey;
+ }
+
+ /**
+ * @param dictionaryKey the dictionaryKey to set
+ */
+ public void setDictionaryKey(byte[] dictionaryKey) {
+ this.dictionaryKey = dictionaryKey;
+ }
+
+ /**
+ * @param noDictionaryKeys the noDictionaryKeys to set
+ */
+ public void setNoDictionaryKeys(byte[][] noDictionaryKeys) {
+ this.noDictionaryKeys = noDictionaryKeys;
+ }
+
+ /**
+ * to get the no dictionary column data
+ *
+ * @param index of the no dictionary key
+ * @return no dictionary key for the index
+ */
+ public byte[] getNoDictionaryKeyByIndex(int index) {
+ return this.noDictionaryKeys[index];
+ }
+
+ /**
+ * to get the no dictionary column data
+ *
+ * @param index of the no dictionary key
+ * @return no dictionary key for the index
+ */
+ public byte[] getComplexTypeByIndex(int index) {
+ return this.complexTypesKeys[index];
+ }
+
+ /**
+ * to generate the hash code
+ */
+ @Override public int hashCode() {
+ // first generate the has code of the dictionary column
+ int len = dictionaryKey.length;
+ int result = 1;
+ for (int j = 0; j < len; j++) {
+ result = 31 * result + dictionaryKey[j];
+ }
+ // then no dictionary column
+ for (byte[] directSurrogateValue : noDictionaryKeys) {
+ for (int i = 0; i < directSurrogateValue.length; i++) {
+ result = 31 * result + directSurrogateValue[i];
+ }
+ }
+ // then for complex type
+ for (byte[] complexTypeKey : complexTypesKeys) {
+ for (int i = 0; i < complexTypeKey.length; i++) {
+ result = 31 * result + complexTypeKey[i];
+ }
+ }
+ return result;
+ }
+
+ /**
+ * to validate the two
+ *
+ * @param other object
+ */
+ @Override public boolean equals(Object other) {
+ if (null == other || !(other instanceof ByteArrayWrapper)) {
+ return false;
+ }
+ boolean result = false;
+ // Comparison will be as follows
+ // first compare the no dictionary column
+ // if it is not equal then return false
+ // if it is equal then compare the complex column
+ // if it is also equal then compare dictionary column
+ byte[][] noDictionaryKeysOther = ((ByteArrayWrapper) other).noDictionaryKeys;
+ if (noDictionaryKeysOther.length != noDictionaryKeys.length) {
+ return false;
+ } else {
+ for (int i = 0; i < noDictionaryKeys.length; i++) {
+ result = UnsafeComparer.INSTANCE.equals(noDictionaryKeys[i], noDictionaryKeysOther[i]);
+ if (!result) {
+ return false;
+ }
+ }
+ }
+
+ byte[][] complexTypesKeysOther = ((ByteArrayWrapper) other).complexTypesKeys;
+ if (complexTypesKeysOther.length != complexTypesKeys.length) {
+ return false;
+ } else {
+ for (int i = 0; i < complexTypesKeys.length; i++) {
+ result = UnsafeComparer.INSTANCE.equals(complexTypesKeys[i], complexTypesKeysOther[i]);
+ if (!result) {
+ return false;
+ }
+ }
+ }
+
+ return UnsafeComparer.INSTANCE.equals(dictionaryKey, ((ByteArrayWrapper) other).dictionaryKey);
+ }
+
+ /**
+ * Compare method for ByteArrayWrapper class this will used to compare Two
+ * ByteArrayWrapper data object, basically it will compare two byte array
+ *
+ * @param other ArrayWrapper Object
+ */
+ @Override public int compareTo(ByteArrayWrapper other) {
+ // compare will be as follows
+ //compare dictionary column
+ // then no dictionary column
+ // then complex type column data
+ int compareTo = UnsafeComparer.INSTANCE.compareTo(dictionaryKey, other.dictionaryKey);
+ if (compareTo == 0) {
+ for (int i = 0; i < noDictionaryKeys.length; i++) {
+ compareTo =
+ UnsafeComparer.INSTANCE.compareTo(noDictionaryKeys[i], other.noDictionaryKeys[i]);
+ if (compareTo != 0) {
+ return compareTo;
+ }
+ }
+ }
+ if (compareTo == 0) {
+ for (int i = 0; i < complexTypesKeys.length; i++) {
+ compareTo =
+ UnsafeComparer.INSTANCE.compareTo(complexTypesKeys[i], other.complexTypesKeys[i]);
+ if (compareTo != 0) {
+ return compareTo;
+ }
+ }
+ }
+ return compareTo;
+ }
+
+ /**
+ * @return the complexTypesKeys
+ */
+ public byte[][] getComplexTypesKeys() {
+ return complexTypesKeys;
+ }
+
+ /**
+ * @param complexTypesKeys the complexTypesKeys to set
+ */
+ public void setComplexTypesKeys(byte[][] complexTypesKeys) {
+ this.complexTypesKeys = complexTypesKeys;
+ }
+
+ /**
+ * @return
+ */
+ public byte[] getImplicitColumnByteArray() {
+ return implicitColumnByteArray;
+ }
+
+ /**
+ * @param implicitColumnByteArray
+ */
+ public void setImplicitColumnByteArray(byte[] implicitColumnByteArray) {
+ this.implicitColumnByteArray = implicitColumnByteArray;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/service/CarbonCommonFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/service/CarbonCommonFactory.java b/core/src/main/java/org/apache/carbondata/core/service/CarbonCommonFactory.java
new file mode 100644
index 0000000..3c33867
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/service/CarbonCommonFactory.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.service;
+
+import org.apache.carbondata.core.service.impl.ColumnUniqueIdGenerator;
+import org.apache.carbondata.core.service.impl.DictionaryFactory;
+import org.apache.carbondata.core.service.impl.PathFactory;
+import org.apache.carbondata.core.service.ColumnUniqueIdService;
+import org.apache.carbondata.core.service.DictionaryService;
+import org.apache.carbondata.core.service.PathService;
+
+/**
+ * Interface to get services
+ */
+public class CarbonCommonFactory {
+
+ /**
+ * @return dictionary service
+ */
+ public static DictionaryService getDictionaryService() {
+ return DictionaryFactory.getInstance();
+ }
+
+ /**
+ * @return path service
+ */
+ public static PathService getPathService() {
+ return PathFactory.getInstance();
+ }
+
+ /**
+ * @return unique id generator
+ */
+ public static ColumnUniqueIdService getColumnUniqueIdGenerator() {
+ return ColumnUniqueIdGenerator.getInstance();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/service/impl/ColumnUniqueIdGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/service/impl/ColumnUniqueIdGenerator.java b/core/src/main/java/org/apache/carbondata/core/service/impl/ColumnUniqueIdGenerator.java
new file mode 100644
index 0000000..b79f9db
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/service/impl/ColumnUniqueIdGenerator.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.service.impl;
+
+import java.util.UUID;
+
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.service.ColumnUniqueIdService;
+
+/**
+ * It returns unique id given column
+ */
+public class ColumnUniqueIdGenerator implements ColumnUniqueIdService {
+
+ private static ColumnUniqueIdService columnUniqueIdService = new ColumnUniqueIdGenerator();
+
+ @Override public String generateUniqueId(String databaseName, ColumnSchema columnSchema) {
+ return UUID.randomUUID().toString();
+ }
+
+ public static ColumnUniqueIdService getInstance() {
+ return columnUniqueIdService;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/service/impl/DictionaryFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/service/impl/DictionaryFactory.java b/core/src/main/java/org/apache/carbondata/core/service/impl/DictionaryFactory.java
new file mode 100644
index 0000000..d9a6d19
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/service/impl/DictionaryFactory.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.service.impl;
+
+import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader;
+import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReaderImpl;
+import org.apache.carbondata.core.reader.CarbonDictionaryReader;
+import org.apache.carbondata.core.reader.CarbonDictionaryReaderImpl;
+import org.apache.carbondata.core.reader.sortindex.CarbonDictionarySortIndexReader;
+import org.apache.carbondata.core.reader.sortindex.CarbonDictionarySortIndexReaderImpl;
+import org.apache.carbondata.core.service.DictionaryService;
+import org.apache.carbondata.core.writer.CarbonDictionaryWriter;
+import org.apache.carbondata.core.writer.CarbonDictionaryWriterImpl;
+import org.apache.carbondata.core.writer.sortindex.CarbonDictionarySortIndexWriter;
+import org.apache.carbondata.core.writer.sortindex.CarbonDictionarySortIndexWriterImpl;
+
+/**
+ * service to get dictionary reader and writer
+ */
+public class DictionaryFactory implements DictionaryService {
+
+ private static DictionaryService dictService = new DictionaryFactory();
+
+ /**
+ * get dictionary writer
+ *
+ * @param carbonTableIdentifier
+ * @param columnIdentifier
+ * @param carbonStorePath
+ * @return
+ */
+ @Override public CarbonDictionaryWriter getDictionaryWriter(
+ CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier,
+ String carbonStorePath) {
+ return new CarbonDictionaryWriterImpl(carbonStorePath, carbonTableIdentifier, columnIdentifier);
+ }
+
+ /**
+ * get dictionary sort index writer
+ *
+ * @param carbonTableIdentifier
+ * @param columnIdentifier
+ * @param carbonStorePath
+ * @return
+ */
+ @Override public CarbonDictionarySortIndexWriter getDictionarySortIndexWriter(
+ CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier,
+ String carbonStorePath) {
+ return new CarbonDictionarySortIndexWriterImpl(carbonTableIdentifier, columnIdentifier,
+ carbonStorePath);
+ }
+
+ /**
+ * get dictionary metadata reader
+ *
+ * @param carbonTableIdentifier
+ * @param columnIdentifier
+ * @param carbonStorePath
+ * @return
+ */
+ @Override public CarbonDictionaryMetadataReader getDictionaryMetadataReader(
+ CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier,
+ String carbonStorePath) {
+ return new CarbonDictionaryMetadataReaderImpl(carbonStorePath, carbonTableIdentifier,
+ columnIdentifier);
+ }
+
+ /**
+ * get dictionary reader
+ *
+ * @param carbonTableIdentifier
+ * @param columnIdentifier
+ * @param carbonStorePath
+ * @return
+ */
+ @Override public CarbonDictionaryReader getDictionaryReader(
+ CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier,
+ String carbonStorePath) {
+ return new CarbonDictionaryReaderImpl(carbonStorePath, carbonTableIdentifier, columnIdentifier);
+ }
+
+ /**
+ * get dictionary sort index reader
+ *
+ * @param carbonTableIdentifier
+ * @param columnIdentifier
+ * @param carbonStorePath
+ * @return
+ */
+ @Override public CarbonDictionarySortIndexReader getDictionarySortIndexReader(
+ CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier,
+ String carbonStorePath) {
+ return new CarbonDictionarySortIndexReaderImpl(carbonTableIdentifier, columnIdentifier,
+ carbonStorePath);
+ }
+
+ public static DictionaryService getInstance() {
+ return dictService;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/service/impl/PathFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/service/impl/PathFactory.java b/core/src/main/java/org/apache/carbondata/core/service/impl/PathFactory.java
new file mode 100644
index 0000000..9cd4b41
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/service/impl/PathFactory.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.service.impl;
+
+import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.path.CarbonStorePath;
+import org.apache.carbondata.core.path.CarbonTablePath;
+import org.apache.carbondata.core.service.PathService;
+
+/**
+ * Create helper to get path details
+ */
+public class PathFactory implements PathService {
+
+ private static PathService pathService = new PathFactory();
+
+ /**
+ * Return store path related to tables
+ */
+ @Override public CarbonTablePath getCarbonTablePath(
+ String storeLocation, CarbonTableIdentifier tableIdentifier) {
+ return CarbonStorePath.getCarbonTablePath(storeLocation, tableIdentifier);
+ }
+
+ public static PathService getInstance() {
+ return pathService;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java
index 26a4778..317d44a 100644
--- a/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java
@@ -22,8 +22,6 @@ import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
-import org.apache.carbondata.common.iudprocessor.iuddata.BlockMappingVO;
-import org.apache.carbondata.common.iudprocessor.iuddata.RowCountDetailsVO;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.AbsoluteTableIdentifier;
@@ -31,10 +29,12 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.load.LoadMetadataDetails;
+import org.apache.carbondata.core.update.data.BlockMappingVO;
+import org.apache.carbondata.core.update.data.RowCountDetailsVO;
import org.apache.carbondata.core.updatestatus.SegmentStatusManager;
import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager;
import org.apache.carbondata.core.util.CarbonProperties;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/update/UpdateVO.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/update/UpdateVO.java b/core/src/main/java/org/apache/carbondata/core/update/UpdateVO.java
deleted file mode 100644
index ad744c4..0000000
--- a/core/src/main/java/org/apache/carbondata/core/update/UpdateVO.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.update;
-
-import java.io.Serializable;
-
-/**
- * VO class for storing details related to Update operation.
- */
-public class UpdateVO implements Serializable {
- private static final long serialVersionUID = 1L;
-
- private Long factTimestamp;
-
- private Long updateDeltaStartTimestamp;
-
- private String segmentId;
-
- public Long getLatestUpdateTimestamp() {
- return latestUpdateTimestamp;
- }
-
- public void setLatestUpdateTimestamp(Long latestUpdateTimestamp) {
- this.latestUpdateTimestamp = latestUpdateTimestamp;
- }
-
- private Long latestUpdateTimestamp;
-
- public Long getFactTimestamp() {
- return factTimestamp;
- }
-
- public void setFactTimestamp(Long factTimestamp) {
- this.factTimestamp = factTimestamp;
- }
-
- public Long getUpdateDeltaStartTimestamp() {
- return updateDeltaStartTimestamp;
- }
-
- public void setUpdateDeltaStartTimestamp(Long updateDeltaStartTimestamp) {
- this.updateDeltaStartTimestamp = updateDeltaStartTimestamp;
- }
-
- @Override public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
- UpdateVO updateVO = (UpdateVO) o;
- if (factTimestamp != null ?
- !factTimestamp.equals(updateVO.factTimestamp) :
- updateVO.factTimestamp != null) {
- return false;
- }
- if (updateDeltaStartTimestamp != null ?
- !updateDeltaStartTimestamp.equals(updateVO.updateDeltaStartTimestamp) :
- updateVO.updateDeltaStartTimestamp != null) {
- return false;
- }
- return latestUpdateTimestamp != null ?
- latestUpdateTimestamp.equals(updateVO.latestUpdateTimestamp) :
- updateVO.latestUpdateTimestamp == null;
-
- }
-
- @Override public int hashCode() {
- int result = factTimestamp != null ? factTimestamp.hashCode() : 0;
- result = 31 * result + (updateDeltaStartTimestamp != null ?
- updateDeltaStartTimestamp.hashCode() :
- 0);
- result = 31 * result + (latestUpdateTimestamp != null ? latestUpdateTimestamp.hashCode() : 0);
- return result;
- }
-
- /**
- * This will return the update timestamp if its present or it will return the fact timestamp.
- * @return
- */
- public Long getCreatedOrUpdatedTimeStamp() {
- if (null == latestUpdateTimestamp) {
- return factTimestamp;
- }
- return latestUpdateTimestamp;
- }
-
- public String getSegmentId() {
- return segmentId;
- }
-
- public void setSegmentId(String segmentId) {
- this.segmentId = segmentId;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/update/data/BlockMappingVO.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/update/data/BlockMappingVO.java b/core/src/main/java/org/apache/carbondata/core/update/data/BlockMappingVO.java
new file mode 100644
index 0000000..244c883
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/update/data/BlockMappingVO.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.update.data;
+
+import java.util.Map;
+
+/**
+ * VO class to store the details of segment and block count , block and its row count.
+ */
+public class BlockMappingVO {
+
+ private Map<String, Long> blockRowCountMapping ;
+
+ private Map<String, Long> segmentNumberOfBlockMapping ;
+
+ private Map<String, RowCountDetailsVO> completeBlockRowDetailVO;
+
+ public void setCompleteBlockRowDetailVO(Map<String, RowCountDetailsVO> completeBlockRowDetailVO) {
+ this.completeBlockRowDetailVO = completeBlockRowDetailVO;
+ }
+
+ public Map<String, RowCountDetailsVO> getCompleteBlockRowDetailVO() {
+ return completeBlockRowDetailVO;
+ }
+
+ public Map<String, Long> getBlockRowCountMapping() {
+ return blockRowCountMapping;
+ }
+
+ public Map<String, Long> getSegmentNumberOfBlockMapping() {
+ return segmentNumberOfBlockMapping;
+ }
+
+ public BlockMappingVO(Map<String, Long> blockRowCountMapping,
+ Map<String, Long> segmentNumberOfBlockMapping) {
+ this.blockRowCountMapping = blockRowCountMapping;
+ this.segmentNumberOfBlockMapping = segmentNumberOfBlockMapping;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/update/data/BlockletDeleteDeltaCacheLoader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/update/data/BlockletDeleteDeltaCacheLoader.java b/core/src/main/java/org/apache/carbondata/core/update/data/BlockletDeleteDeltaCacheLoader.java
new file mode 100644
index 0000000..2015663
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/update/data/BlockletDeleteDeltaCacheLoader.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.update.data;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager;
+
+/**
+ * This class is responsible for loading delete delta file cache based on
+ * blocklet id of a particular block
+ */
+public class BlockletDeleteDeltaCacheLoader implements DeleteDeltaCacheLoaderIntf {
+ private String blockletID;
+ private DataRefNode blockletNode;
+ private AbsoluteTableIdentifier absoluteIdentifier;
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(BlockletDeleteDeltaCacheLoader.class.getName());
+
+ public BlockletDeleteDeltaCacheLoader(String blockletID,
+ DataRefNode blockletNode, AbsoluteTableIdentifier absoluteIdentifier) {
+ this.blockletID = blockletID;
+ this.blockletNode = blockletNode;
+ this.absoluteIdentifier= absoluteIdentifier;
+ }
+
+ /**
+ * This method will load the delete delta cache based on blocklet id of particular block with
+ * the help of SegmentUpdateStatusManager.
+ */
+ public void loadDeleteDeltaFileDataToCache() {
+ SegmentUpdateStatusManager segmentUpdateStatusManager =
+ new SegmentUpdateStatusManager(absoluteIdentifier);
+ int[] deleteDeltaFileData = null;
+ BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache = null;
+ if (null == blockletNode.getDeleteDeltaDataCache()) {
+ try {
+ deleteDeltaFileData = segmentUpdateStatusManager.getDeleteDeltaDataFromAllFiles(blockletID);
+ deleteDeltaDataCache = new BlockletLevelDeleteDeltaDataCache(deleteDeltaFileData,
+ segmentUpdateStatusManager.getTimestampForRefreshCache(blockletID, null));
+ } catch (Exception e) {
+ LOGGER.debug("Unable to retrieve delete delta files");
+ }
+ } else {
+ deleteDeltaDataCache = blockletNode.getDeleteDeltaDataCache();
+ // if already cache is present then validate the cache using timestamp
+ String cacheTimeStamp = segmentUpdateStatusManager
+ .getTimestampForRefreshCache(blockletID, deleteDeltaDataCache.getCacheTimeStamp());
+ if (null != cacheTimeStamp) {
+ try {
+ deleteDeltaFileData =
+ segmentUpdateStatusManager.getDeleteDeltaDataFromAllFiles(blockletID);
+ deleteDeltaDataCache = new BlockletLevelDeleteDeltaDataCache(deleteDeltaFileData,
+ segmentUpdateStatusManager.getTimestampForRefreshCache(blockletID, cacheTimeStamp));
+ } catch (Exception e) {
+ LOGGER.debug("Unable to retrieve delete delta files");
+ }
+ }
+ }
+ blockletNode.setDeleteDeltaDataCache(deleteDeltaDataCache);
+ }
+}
[48/57] [abbrv] incubator-carbondata git commit: fix style
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneLong.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneLong.java
new file mode 100644
index 0000000..04bab54
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneLong.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression.none;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastore.compression.Compressor;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNoneLong extends ValueCompressionHolder<long[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNoneLong.class.getName());
+ /**
+ * longCompressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+ /**
+ * value.
+ */
+ protected long[] value;
+
+ private DataType actualDataType;
+
+ private MeasureDataChunkStore<long[]> measureChunkStore;
+
+ public CompressionNoneLong(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
+ @Override public void setValue(long[] value) { this.value = value; }
+
+ @Override public long[] getValue() { return this.value; }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_LONG, value);
+ }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] data, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, data, offset, length);
+ setUncompressedValues(value);
+ }
+
+ @Override public void setValueInBytes(byte[] byteValue) {
+ ByteBuffer buffer = ByteBuffer.wrap(byteValue);
+ this.value = ValueCompressionUtil.convertToLongArray(buffer, byteValue.length);
+ }
+
+ @Override public long getLongValue(int index) {
+ return measureChunkStore.getLong(index);
+ }
+
+ @Override public double getDoubleValue(int index) {
+ return measureChunkStore.getLong(index);
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException("Get big decimal is not supported");
+ }
+
+ private void setUncompressedValues(long[] data) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_LONG, data.length);
+ this.measureChunkStore.putData(data);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneShort.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneShort.java
new file mode 100644
index 0000000..b84c562
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/none/CompressionNoneShort.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression.none;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastore.compression.Compressor;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNoneShort extends ValueCompressionHolder<short[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNoneShort.class.getName());
+
+ /**
+ * shortCompressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+
+ /**
+ * value.
+ */
+ private short[] shortValue;
+
+ private MeasureDataChunkStore<short[]> measureChunkStore;
+
+ private DataType actualDataType;
+
+ public CompressionNoneShort(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
+ @Override public void setValue(short[] shortValue) {
+ this.shortValue = shortValue;
+ }
+
+ @Override public short[] getValue() { return this.shortValue; }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] data, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, data, offset, length);
+ setUncompressedValues(shortValue);
+ }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_SHORT, shortValue);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ ByteBuffer buffer = ByteBuffer.wrap(value);
+ shortValue = ValueCompressionUtil.convertToShortArray(buffer, value.length);
+ }
+
+ @Override public long getLongValue(int index) {
+ return measureChunkStore.getShort(index);
+ }
+
+ @Override public double getDoubleValue(int index) {
+ return measureChunkStore.getShort(index);
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal is not defined for CompressionNonShort");
+ }
+
+ private void setUncompressedValues(short[] data) {
+ this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
+ .getMeasureDataChunkStore(DataType.DATA_SHORT, data.length);
+ this.measureChunkStore.putData(data);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/type/CompressionBigDecimal.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/type/CompressionBigDecimal.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/type/CompressionBigDecimal.java
new file mode 100644
index 0000000..eb0e5d0
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/type/CompressionBigDecimal.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.datastore.compression.type;
+
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.BigDecimalCompressionFinder;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+/**
+ * Big decimal compression
+ */
+public class CompressionBigDecimal<T> extends ValueCompressionHolder<T> {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionBigDecimal.class.getName());
+
+ private BigDecimalCompressionFinder compressionFinder;
+
+ /**
+ * leftPart before decimal
+ */
+ private ValueCompressionHolder leftPart;
+
+ /**
+ * rightPart after decimal
+ */
+ private ValueCompressionHolder rightPart;
+
+ private double divisionFactor;
+
+ private boolean isDecimalPlacesNotZero;
+
+ public CompressionBigDecimal(BigDecimalCompressionFinder compressionFinder,
+ ValueCompressionHolder leftPart, ValueCompressionHolder rightPart) {
+ this.compressionFinder = compressionFinder;
+ this.leftPart = leftPart;
+ this.rightPart = rightPart;
+ }
+
+ @Override public void setValue(T value) {
+ Object[] values = (Object[]) value;
+ leftPart.setValue(values[0]);
+ rightPart.setValue(values[1]);
+ }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] data, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
+ if (decimalPlaces > 0) {
+ this.isDecimalPlacesNotZero = true;
+ }
+ this.divisionFactor = Math.pow(10, decimalPlaces);
+
+ ByteBuffer buffer = ByteBuffer.wrap(data, offset, length);
+ int leftPathLength = buffer.getInt();
+ int rightPartLength = length - leftPathLength - CarbonCommonConstants.INT_SIZE_IN_BYTE;
+ Long[] maxValue = (Long[]) maxValueObject;
+ leftPart.uncompress(compressionFinder.getLeftConvertedDataType(), data,
+ offset + CarbonCommonConstants.INT_SIZE_IN_BYTE, leftPathLength, decimalPlaces,
+ maxValue[0]);
+ rightPart.uncompress(compressionFinder.getRightConvertedDataType(), data,
+ offset + CarbonCommonConstants.INT_SIZE_IN_BYTE + leftPathLength, rightPartLength,
+ decimalPlaces, maxValue[1]);
+ }
+
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException(
+ "Long is not defined for CompressionBigDecimal");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ throw new UnsupportedOperationException(
+ "Double is not defined for CompressionBigDecimal");
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ long leftValue = leftPart.getLongValue(index);
+ long rightValue = 0;
+ if (isDecimalPlacesNotZero) {
+ rightValue = rightPart.getLongValue(index);
+ }
+ String decimalPart = Double.toString(rightValue / this.divisionFactor);
+ String bigdStr = Long.toString(leftValue) + CarbonCommonConstants.POINT + decimalPart
+ .substring(decimalPart.indexOf(".") + 1, decimalPart.length());
+ return new BigDecimal(bigdStr);
+ }
+
+ @Override public T getValue() {
+ Object[] values = new Object[2];
+ values[0] = leftPart;
+ values[1] = rightPart;
+ return (T) values;
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ LOGGER.error("setValueInBytes() is not defined for CompressionBigDecimal");
+ }
+
+ @Override public void compress() {
+ leftPart.compress();
+ rightPart.compress();
+ }
+
+ @Override
+ public byte[] getCompressedData() {
+ byte[] leftdata = leftPart.getCompressedData();
+ byte[] rightdata = rightPart.getCompressedData();
+ ByteBuffer byteBuffer = ByteBuffer
+ .allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE + leftdata.length
+ + rightdata.length);
+ byteBuffer.putInt(leftdata.length);
+ byteBuffer.put(leftdata);
+ byteBuffer.put(rightdata);
+ byteBuffer.flip();
+ return byteBuffer.array();
+ }
+
+ @Override public void freeMemory() {
+ leftPart.freeMemory();
+ rightPart.freeMemory();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/dataholder/CarbonReadDataHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/dataholder/CarbonReadDataHolder.java b/core/src/main/java/org/apache/carbondata/core/datastore/dataholder/CarbonReadDataHolder.java
new file mode 100644
index 0000000..6bd28fc
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/dataholder/CarbonReadDataHolder.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.dataholder;
+
+import java.math.BigDecimal;
+
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+
+// This class is used with Uncompressor to hold the decompressed column chunk in memory
+public class CarbonReadDataHolder {
+
+ private ValueCompressionHolder unCompressValue;
+
+ public CarbonReadDataHolder(ValueCompressionHolder unCompressValue) {
+ this.unCompressValue = unCompressValue;
+ }
+
+ public long getReadableLongValueByIndex(int index) {
+ return this.unCompressValue.getLongValue(index);
+ }
+
+ public BigDecimal getReadableBigDecimalValueByIndex(int index) {
+ return this.unCompressValue.getBigDecimalValue(index);
+ }
+
+ public double getReadableDoubleValueByIndex(int index) {
+ return this.unCompressValue.getDoubleValue(index);
+ }
+
+ public void freeMemory() {
+ unCompressValue.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/dataholder/CarbonWriteDataHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/dataholder/CarbonWriteDataHolder.java b/core/src/main/java/org/apache/carbondata/core/datastore/dataholder/CarbonWriteDataHolder.java
new file mode 100644
index 0000000..23799e8
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/dataholder/CarbonWriteDataHolder.java
@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.dataholder;
+
+public class CarbonWriteDataHolder {
+ /**
+ * doubleValues
+ */
+ private double[] doubleValues;
+
+ /**
+ * longValues
+ */
+ private long[] longValues;
+
+ /**
+ * bigDecimal left part
+ */
+ private long[] bigDecimalLeftValues;
+
+ /**
+ * bigDecimal right part
+ */
+ private long[] bigDecimalRightValues;
+ /**
+ * byteValues
+ */
+ private byte[][] byteValues;
+
+ /**
+ * byteValues for no dictionary and non kettle flow.
+ */
+ private byte[][][] byteValuesForNonDictionary;
+
+ /**
+ * byteValues
+ */
+ private byte[][][] columnByteValues;
+
+ /**
+ * size
+ */
+ private int size;
+
+ /**
+ * totalSize
+ */
+ private int totalSize;
+
+ /**
+ * Method to initialise double array
+ *
+ * @param size
+ */
+ public void initialiseDoubleValues(int size) {
+ if (size < 1) {
+ throw new IllegalArgumentException("Invalid array size");
+ }
+ doubleValues = new double[size];
+ }
+
+ public void reset() {
+ size = 0;
+ totalSize = 0;
+ }
+
+ /**
+ * Method to initialise double array
+ * TODO Remove after kettle flow got removed.
+ *
+ * @param size
+ */
+ public void initialiseByteArrayValues(int size) {
+ if (size < 1) {
+ throw new IllegalArgumentException("Invalid array size");
+ }
+
+ byteValues = new byte[size][];
+ columnByteValues = new byte[size][][];
+ }
+
+ /**
+ * Method to initialise byte array
+ *
+ * @param size
+ */
+ public void initialiseByteArrayValuesWithOutKettle(int size) {
+ if (size < 1) {
+ throw new IllegalArgumentException("Invalid array size");
+ }
+
+ byteValues = new byte[size][];
+ }
+
+ public void initialiseByteArrayValuesForNonDictionary(int size) {
+ if (size < 1) {
+ throw new IllegalArgumentException("Invalid array size");
+ }
+
+ byteValuesForNonDictionary = new byte[size][][];
+ }
+
+ /**
+ * Method to initialise long array
+ *
+ * @param size
+ */
+ public void initialiseLongValues(int size) {
+ if (size < 1) {
+ throw new IllegalArgumentException("Invalid array size");
+ }
+ longValues = new long[size];
+ }
+
+ public void initialiseBigDecimalValues(int size) {
+ if (size < 1) {
+ throw new IllegalArgumentException("Invalid array size");
+ }
+ bigDecimalLeftValues = new long[size];
+ bigDecimalRightValues = new long[size];
+ }
+
+ /**
+ * set double value by index
+ *
+ * @param index
+ * @param value
+ */
+ public void setWritableDoubleValueByIndex(int index, Object value) {
+ doubleValues[index] = (Double) value;
+ size++;
+ }
+
+ /**
+ * set double value by index
+ *
+ * @param index
+ * @param value
+ */
+ public void setWritableLongValueByIndex(int index, Object value) {
+ longValues[index] = (Long) value;
+ size++;
+ }
+
+ /**
+ * set bigdecimal value by index
+ *
+ * @param index
+ * @param value
+ */
+ public void setWritableBigDecimalValueByIndex(int index, long[] value) {
+ bigDecimalLeftValues[index] = value[0];
+ bigDecimalRightValues[index] = value[1];
+ size++;
+ }
+ /**
+ * set byte array value by index
+ *
+ * @param index
+ * @param value
+ */
+ public void setWritableByteArrayValueByIndex(int index, byte[] value) {
+ byteValues[index] = value;
+ size++;
+ if (null != value) totalSize += value.length;
+ }
+
+ public void setWritableNonDictByteArrayValueByIndex(int index, byte[][] value) {
+ byteValuesForNonDictionary[index] = value;
+ size++;
+ if (null != value) totalSize += value.length;
+ }
+
+ /**
+ * set byte array value by index
+ */
+ public void setWritableByteArrayValueByIndex(int index, int mdKeyIndex, Object[] columnData) {
+ int l = 0;
+ columnByteValues[index] = new byte[columnData.length - (mdKeyIndex + 1)][];
+ for (int i = mdKeyIndex + 1; i < columnData.length; i++) {
+ columnByteValues[index][l++] = (byte[]) columnData[i];
+ }
+ }
+
+ /**
+ * Get Writable Double Values
+ */
+ public double[] getWritableDoubleValues() {
+ if (size < doubleValues.length) {
+ double[] temp = new double[size];
+ System.arraycopy(doubleValues, 0, temp, 0, size);
+ doubleValues = temp;
+ }
+ return doubleValues;
+ }
+
+ /**
+ * Get writable byte array values
+ */
+ public byte[] getWritableByteArrayValues() {
+ byte[] temp = new byte[totalSize];
+ int startIndexToCopy = 0;
+ for (int i = 0; i < size; i++) {
+ System.arraycopy(byteValues[i], 0, temp, startIndexToCopy, byteValues[i].length);
+ startIndexToCopy += byteValues[i].length;
+ }
+ return temp;
+ }
+
+ public byte[][] getByteArrayValues() {
+ if (size < byteValues.length) {
+ byte[][] temp = new byte[size][];
+ System.arraycopy(byteValues, 0, temp, 0, size);
+ byteValues = temp;
+ }
+ return byteValues;
+ }
+
+ public byte[][][] getNonDictByteArrayValues() {
+ if (size < byteValuesForNonDictionary.length) {
+ byte[][][] temp = new byte[size][][];
+ System.arraycopy(byteValuesForNonDictionary, 0, temp, 0, size);
+ byteValuesForNonDictionary = temp;
+ }
+ return byteValuesForNonDictionary;
+ }
+
+ /**
+ * Get Writable Double Values
+ *
+ * @return
+ */
+ public long[] getWritableLongValues() {
+ if (size < longValues.length) {
+ long[] temp = new long[size];
+ System.arraycopy(longValues, 0, temp, 0, size);
+ longValues = temp;
+ }
+ return longValues;
+ }
+
+ /**
+ * Get Writable bigdecimal Values
+ *
+ * @return
+ */
+ public long[][] getWritableBigDecimalValues() {
+ long[][] bigDecimalValues = new long[2][];
+ if (size < bigDecimalLeftValues.length) {
+ long[] temp = new long[size];
+ System.arraycopy(bigDecimalLeftValues, 0, temp, 0, size);
+ bigDecimalLeftValues = temp;
+ }
+ if (size < bigDecimalRightValues.length) {
+ long[] temp = new long[size];
+ System.arraycopy(bigDecimalRightValues, 0, temp, 0, size);
+ bigDecimalRightValues = temp;
+ }
+ bigDecimalValues[0]= bigDecimalLeftValues;
+ bigDecimalValues[1] = bigDecimalRightValues;
+ return bigDecimalValues;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
new file mode 100644
index 0000000..01ad646
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AbstractDFSCarbonFile.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.filesystem;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+public abstract class AbstractDFSCarbonFile implements CarbonFile {
+ /**
+ * LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(AbstractDFSCarbonFile.class.getName());
+ protected FileStatus fileStatus;
+ protected FileSystem fs;
+
+ public AbstractDFSCarbonFile(String filePath) {
+ filePath = filePath.replace("\\", "/");
+ Path path = new Path(filePath);
+ try {
+ fs = path.getFileSystem(FileFactory.getConfiguration());
+ fileStatus = fs.getFileStatus(path);
+ } catch (IOException e) {
+ LOGGER.error("Exception occured:" + e.getMessage());
+ }
+ }
+
+ public AbstractDFSCarbonFile(Path path) {
+ try {
+ fs = path.getFileSystem(FileFactory.getConfiguration());
+ fileStatus = fs.getFileStatus(path);
+ } catch (IOException e) {
+ LOGGER.error("Exception occured:" + e.getMessage());
+ }
+ }
+
+ public AbstractDFSCarbonFile(FileStatus fileStatus) {
+ this.fileStatus = fileStatus;
+ }
+
+ @Override public boolean createNewFile() {
+ Path path = fileStatus.getPath();
+ try {
+ return fs.createNewFile(path);
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ @Override public String getAbsolutePath() {
+ return fileStatus.getPath().toString();
+ }
+
+ @Override public String getName() {
+ return fileStatus.getPath().getName();
+ }
+
+ @Override public boolean isDirectory() {
+ return fileStatus.isDirectory();
+ }
+
+ @Override public boolean exists() {
+ try {
+ if (null != fileStatus) {
+ fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
+ return fs.exists(fileStatus.getPath());
+ }
+ } catch (IOException e) {
+ LOGGER.error("Exception occured:" + e.getMessage());
+ }
+ return false;
+ }
+
+ @Override public String getCanonicalPath() {
+ return getAbsolutePath();
+ }
+
+ @Override public String getPath() {
+ return getAbsolutePath();
+ }
+
+ @Override public long getSize() {
+ return fileStatus.getLen();
+ }
+
+ public boolean renameTo(String changetoName) {
+ FileSystem fs;
+ try {
+ fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
+ return fs.rename(fileStatus.getPath(), new Path(changetoName));
+ } catch (IOException e) {
+ LOGGER.error("Exception occured:" + e.getMessage());
+ return false;
+ }
+ }
+
+ public boolean delete() {
+ FileSystem fs;
+ try {
+ fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
+ return fs.delete(fileStatus.getPath(), true);
+ } catch (IOException e) {
+ LOGGER.error("Exception occured:" + e.getMessage());
+ return false;
+ }
+ }
+
+ @Override public long getLastModifiedTime() {
+ return fileStatus.getModificationTime();
+ }
+
+ @Override public boolean setLastModifiedTime(long timestamp) {
+ try {
+ fs.setTimes(fileStatus.getPath(), timestamp, timestamp);
+ } catch (IOException e) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * This method will delete the data in file data from a given offset
+ */
+ @Override public boolean truncate(String fileName, long validDataEndOffset) {
+ DataOutputStream dataOutputStream = null;
+ DataInputStream dataInputStream = null;
+ boolean fileTruncatedSuccessfully = false;
+ // if bytes to read less than 1024 then buffer size should be equal to the given offset
+ int bufferSize = validDataEndOffset > CarbonCommonConstants.BYTE_TO_KB_CONVERSION_FACTOR ?
+ CarbonCommonConstants.BYTE_TO_KB_CONVERSION_FACTOR :
+ (int) validDataEndOffset;
+ // temporary file name
+ String tempWriteFilePath = fileName + CarbonCommonConstants.TEMPWRITEFILEEXTENSION;
+ FileFactory.FileType fileType = FileFactory.getFileType(fileName);
+ try {
+ CarbonFile tempFile = null;
+ // delete temporary file if it already exists at a given path
+ if (FileFactory.isFileExist(tempWriteFilePath, fileType)) {
+ tempFile = FileFactory.getCarbonFile(tempWriteFilePath, fileType);
+ tempFile.delete();
+ }
+ // create new temporary file
+ FileFactory.createNewFile(tempWriteFilePath, fileType);
+ tempFile = FileFactory.getCarbonFile(tempWriteFilePath, fileType);
+ byte[] buff = new byte[bufferSize];
+ dataInputStream = FileFactory.getDataInputStream(fileName, fileType);
+ // read the data
+ int read = dataInputStream.read(buff, 0, buff.length);
+ dataOutputStream = FileFactory.getDataOutputStream(tempWriteFilePath, fileType);
+ dataOutputStream.write(buff, 0, read);
+ long remaining = validDataEndOffset - read;
+ // anytime we should not cross the offset to be read
+ while (remaining > 0) {
+ if (remaining > bufferSize) {
+ buff = new byte[bufferSize];
+ } else {
+ buff = new byte[(int) remaining];
+ }
+ read = dataInputStream.read(buff, 0, buff.length);
+ dataOutputStream.write(buff, 0, read);
+ remaining = remaining - read;
+ }
+ CarbonUtil.closeStreams(dataInputStream, dataOutputStream);
+ // rename the temp file to original file
+ tempFile.renameForce(fileName);
+ fileTruncatedSuccessfully = true;
+ } catch (IOException e) {
+ LOGGER.error("Exception occured while truncating the file " + e.getMessage());
+ } finally {
+ CarbonUtil.closeStreams(dataOutputStream, dataInputStream);
+ }
+ return fileTruncatedSuccessfully;
+ }
+
+ /**
+ * This method will be used to check whether a file has been modified or not
+ *
+ * @param fileTimeStamp time to be compared with latest timestamp of file
+ * @param endOffset file length to be compared with current length of file
+ * @return
+ */
+ @Override public boolean isFileModified(long fileTimeStamp, long endOffset) {
+ boolean isFileModified = false;
+ if (getLastModifiedTime() > fileTimeStamp || getSize() > endOffset) {
+ isFileModified = true;
+ }
+ return isFileModified;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFile.java
new file mode 100644
index 0000000..51fcd53
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFile.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.filesystem;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+
+
+
+public class AlluxioCarbonFile extends AbstractDFSCarbonFile {
+ /**
+ * LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(AlluxioCarbonFile.class.getName());
+
+ public AlluxioCarbonFile(String filePath) {
+ super(filePath);
+ }
+
+ public AlluxioCarbonFile(Path path) {
+ super(path);
+ }
+
+ public AlluxioCarbonFile(FileStatus fileStatus) {
+ super(fileStatus);
+ }
+
+ /**
+ * @param listStatus
+ * @return
+ */
+ private CarbonFile[] getFiles(FileStatus[] listStatus) {
+ if (listStatus == null) {
+ return new CarbonFile[0];
+ }
+ CarbonFile[] files = new CarbonFile[listStatus.length];
+ for (int i = 0; i < files.length; i++) {
+ files[i] = new AlluxioCarbonFile(listStatus[i]);
+ }
+ return files;
+ }
+
+ @Override
+ public CarbonFile[] listFiles() {
+ FileStatus[] listStatus = null;
+ try {
+ if (null != fileStatus && fileStatus.isDirectory()) {
+ Path path = fileStatus.getPath();
+ listStatus = path.getFileSystem(FileFactory.getConfiguration()).listStatus(path);
+ } else {
+ return null;
+ }
+ } catch (IOException e) {
+ LOGGER.error("Exception occured: " + e.getMessage());
+ return new CarbonFile[0];
+ }
+ return getFiles(listStatus);
+ }
+
+ @Override
+ public CarbonFile[] listFiles(final CarbonFileFilter fileFilter) {
+ CarbonFile[] files = listFiles();
+ if (files != null && files.length >= 1) {
+ List<CarbonFile> fileList = new ArrayList<CarbonFile>(files.length);
+ for (int i = 0; i < files.length; i++) {
+ if (fileFilter.accept(files[i])) {
+ fileList.add(files[i]);
+ }
+ }
+ if (fileList.size() >= 1) {
+ return fileList.toArray(new CarbonFile[fileList.size()]);
+ } else {
+ return new CarbonFile[0];
+ }
+ }
+ return files;
+ }
+
+ @Override
+ public CarbonFile getParentFile() {
+ Path parent = fileStatus.getPath().getParent();
+ return null == parent ? null : new AlluxioCarbonFile(parent);
+ }
+
+ @Override
+ public boolean renameForce(String changetoName) {
+ FileSystem fs;
+ try {
+ fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
+ if (fs instanceof DistributedFileSystem) {
+ ((DistributedFileSystem) fs).rename(fileStatus.getPath(), new Path(changetoName),
+ org.apache.hadoop.fs.Options.Rename.OVERWRITE);
+ return true;
+ } else {
+ return false;
+ }
+ } catch (IOException e) {
+ LOGGER.error("Exception occured: " + e.getMessage());
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/CarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/CarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/CarbonFile.java
new file mode 100644
index 0000000..a00f633
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/CarbonFile.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.filesystem;
+
+public interface CarbonFile {
+
+ String getAbsolutePath();
+
+ CarbonFile[] listFiles(CarbonFileFilter fileFilter);
+
+ CarbonFile[] listFiles();
+
+ String getName();
+
+ boolean isDirectory();
+
+ boolean exists();
+
+ String getCanonicalPath();
+
+ CarbonFile getParentFile();
+
+ String getPath();
+
+ long getSize();
+
+ boolean renameTo(String changetoName);
+
+ boolean renameForce(String changetoName);
+
+ boolean delete();
+
+ boolean createNewFile();
+
+ long getLastModifiedTime();
+
+ boolean setLastModifiedTime(long timestamp);
+
+ boolean truncate(String fileName, long validDataEndOffset);
+
+ /**
+ * This method will be used to check whether a file has been modified or not
+ *
+ * @param fileTimeStamp time to be compared with latest timestamp of file
+ * @param endOffset file length to be compared with current length of file
+ * @return
+ */
+ boolean isFileModified(long fileTimeStamp, long endOffset);
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/CarbonFileFilter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/CarbonFileFilter.java b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/CarbonFileFilter.java
new file mode 100644
index 0000000..dc3df36
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/CarbonFileFilter.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.filesystem;
+
+public interface CarbonFileFilter {
+ boolean accept(CarbonFile file);
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java
new file mode 100644
index 0000000..7137087
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFile.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.filesystem;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+
+public class HDFSCarbonFile extends AbstractDFSCarbonFile {
+ /**
+ * LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(HDFSCarbonFile.class.getName());
+
+ public HDFSCarbonFile(String filePath) {
+ super(filePath);
+ }
+
+ public HDFSCarbonFile(Path path) {
+ super(path);
+ }
+
+ public HDFSCarbonFile(FileStatus fileStatus) {
+ super(fileStatus);
+ }
+
+ /**
+ * @param listStatus
+ * @return
+ */
+ private CarbonFile[] getFiles(FileStatus[] listStatus) {
+ if (listStatus == null) {
+ return new CarbonFile[0];
+ }
+ CarbonFile[] files = new CarbonFile[listStatus.length];
+ for (int i = 0; i < files.length; i++) {
+ files[i] = new HDFSCarbonFile(listStatus[i]);
+ }
+ return files;
+ }
+
+ @Override
+ public CarbonFile[] listFiles() {
+ FileStatus[] listStatus = null;
+ try {
+ if (null != fileStatus && fileStatus.isDirectory()) {
+ Path path = fileStatus.getPath();
+ listStatus = path.getFileSystem(FileFactory.getConfiguration()).listStatus(path);
+ } else {
+ return null;
+ }
+ } catch (IOException e) {
+ LOGGER.error("Exception occured: " + e.getMessage());
+ return new CarbonFile[0];
+ }
+ return getFiles(listStatus);
+ }
+
+ @Override
+ public CarbonFile[] listFiles(final CarbonFileFilter fileFilter) {
+ CarbonFile[] files = listFiles();
+ if (files != null && files.length >= 1) {
+ List<CarbonFile> fileList = new ArrayList<CarbonFile>(files.length);
+ for (int i = 0; i < files.length; i++) {
+ if (fileFilter.accept(files[i])) {
+ fileList.add(files[i]);
+ }
+ }
+ if (fileList.size() >= 1) {
+ return fileList.toArray(new CarbonFile[fileList.size()]);
+ } else {
+ return new CarbonFile[0];
+ }
+ }
+ return files;
+ }
+
+ @Override
+ public CarbonFile getParentFile() {
+ Path parent = fileStatus.getPath().getParent();
+ return null == parent ? null : new HDFSCarbonFile(parent);
+ }
+
+ @Override
+ public boolean renameForce(String changetoName) {
+ FileSystem fs;
+ try {
+ fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
+ if (fs instanceof DistributedFileSystem) {
+ ((DistributedFileSystem) fs).rename(fileStatus.getPath(), new Path(changetoName),
+ org.apache.hadoop.fs.Options.Rename.OVERWRITE);
+ return true;
+ } else {
+ return false;
+ }
+ } catch (IOException e) {
+ LOGGER.error("Exception occured: " + e.getMessage());
+ return false;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
new file mode 100644
index 0000000..a6dbfa7
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.filesystem;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+import org.apache.hadoop.fs.Path;
+
+public class LocalCarbonFile implements CarbonFile {
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(LocalCarbonFile.class.getName());
+ private File file;
+
+ public LocalCarbonFile(String filePath) {
+ Path pathWithoutSchemeAndAuthority = Path.getPathWithoutSchemeAndAuthority(new Path(filePath));
+ file = new File(pathWithoutSchemeAndAuthority.toString());
+ }
+
+ public LocalCarbonFile(File file) {
+ this.file = file;
+ }
+
+ @Override public String getAbsolutePath() {
+ return file.getAbsolutePath();
+ }
+
+ @Override public CarbonFile[] listFiles(final CarbonFileFilter fileFilter) {
+ if (!file.isDirectory()) {
+ return null;
+ }
+
+ File[] files = file.listFiles(new FileFilter() {
+
+ @Override public boolean accept(File pathname) {
+ return fileFilter.accept(new LocalCarbonFile(pathname));
+ }
+ });
+
+ if (files == null) {
+ return new CarbonFile[0];
+ }
+
+ CarbonFile[] carbonFiles = new CarbonFile[files.length];
+
+ for (int i = 0; i < carbonFiles.length; i++) {
+ carbonFiles[i] = new LocalCarbonFile(files[i]);
+ }
+
+ return carbonFiles;
+ }
+
+ @Override public String getName() {
+ return file.getName();
+ }
+
+ @Override public boolean isDirectory() {
+ return file.isDirectory();
+ }
+
+ @Override public boolean exists() {
+ if (file != null) {
+ return file.exists();
+ }
+ return false;
+ }
+
+ @Override public String getCanonicalPath() {
+ try {
+ return file.getCanonicalPath();
+ } catch (IOException e) {
+ LOGGER
+ .error(e, "Exception occured" + e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public CarbonFile getParentFile() {
+ return new LocalCarbonFile(file.getParentFile());
+ }
+
+ @Override public String getPath() {
+ return file.getPath();
+ }
+
+ @Override public long getSize() {
+ return file.length();
+ }
+
+ public boolean renameTo(String changetoName) {
+ return file.renameTo(new File(changetoName));
+ }
+
+ public boolean delete() {
+ return file.delete();
+ }
+
+ @Override public CarbonFile[] listFiles() {
+
+ if (!file.isDirectory()) {
+ return null;
+ }
+ File[] files = file.listFiles();
+ if (files == null) {
+ return new CarbonFile[0];
+ }
+ CarbonFile[] carbonFiles = new CarbonFile[files.length];
+ for (int i = 0; i < carbonFiles.length; i++) {
+ carbonFiles[i] = new LocalCarbonFile(files[i]);
+ }
+
+ return carbonFiles;
+
+ }
+
+ @Override public boolean createNewFile() {
+ try {
+ return file.createNewFile();
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ @Override public long getLastModifiedTime() {
+ return file.lastModified();
+ }
+
+ @Override public boolean setLastModifiedTime(long timestamp) {
+ return file.setLastModified(timestamp);
+ }
+
+ /**
+ * This method will delete the data in file data from a given offset
+ */
+ @Override public boolean truncate(String fileName, long validDataEndOffset) {
+ FileChannel source = null;
+ FileChannel destination = null;
+ boolean fileTruncatedSuccessfully = false;
+ // temporary file name
+ String tempWriteFilePath = fileName + CarbonCommonConstants.TEMPWRITEFILEEXTENSION;
+ FileFactory.FileType fileType = FileFactory.getFileType(fileName);
+ try {
+ CarbonFile tempFile = null;
+ // delete temporary file if it already exists at a given path
+ if (FileFactory.isFileExist(tempWriteFilePath, fileType)) {
+ tempFile = FileFactory.getCarbonFile(tempWriteFilePath, fileType);
+ tempFile.delete();
+ }
+ // create new temporary file
+ FileFactory.createNewFile(tempWriteFilePath, fileType);
+ tempFile = FileFactory.getCarbonFile(tempWriteFilePath, fileType);
+ source = new FileInputStream(fileName).getChannel();
+ destination = new FileOutputStream(tempWriteFilePath).getChannel();
+ long read = destination.transferFrom(source, 0, validDataEndOffset);
+ long totalBytesRead = read;
+ long remaining = validDataEndOffset - totalBytesRead;
+ // read till required data offset is not reached
+ while (remaining > 0) {
+ read = destination.transferFrom(source, totalBytesRead, remaining);
+ totalBytesRead = totalBytesRead + read;
+ remaining = remaining - totalBytesRead;
+ }
+ CarbonUtil.closeStreams(source, destination);
+ // rename the temp file to original file
+ tempFile.renameForce(fileName);
+ fileTruncatedSuccessfully = true;
+ } catch (IOException e) {
+ LOGGER.error("Exception occured while truncating the file " + e.getMessage());
+ } finally {
+ CarbonUtil.closeStreams(source, destination);
+ }
+ return fileTruncatedSuccessfully;
+ }
+
+ /**
+ * This method will be used to check whether a file has been modified or not
+ *
+ * @param fileTimeStamp time to be compared with latest timestamp of file
+ * @param endOffset file length to be compared with current length of file
+ * @return
+ */
+ @Override public boolean isFileModified(long fileTimeStamp, long endOffset) {
+ boolean isFileModified = false;
+ if (getLastModifiedTime() > fileTimeStamp || getSize() > endOffset) {
+ isFileModified = true;
+ }
+ return isFileModified;
+ }
+
+ @Override public boolean renameForce(String changetoName) {
+ File destFile = new File(changetoName);
+ if (destFile.exists()) {
+ if (destFile.delete()) {
+ return file.renameTo(new File(changetoName));
+ }
+ }
+
+ return file.renameTo(new File(changetoName));
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/ViewFSCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/ViewFSCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/ViewFSCarbonFile.java
new file mode 100644
index 0000000..c7dd192
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/ViewFSCarbonFile.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.datastore.filesystem;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.viewfs.ViewFileSystem;
+
+public class ViewFSCarbonFile extends AbstractDFSCarbonFile {
+ /**
+ * LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(ViewFSCarbonFile.class.getName());
+
+ public ViewFSCarbonFile(String filePath) {
+ super(filePath);
+ }
+
+ public ViewFSCarbonFile(Path path) {
+ super(path);
+ }
+
+ public ViewFSCarbonFile(FileStatus fileStatus) {
+ super(fileStatus);
+ }
+
+ /**
+ * @param listStatus
+ * @return
+ */
+ private CarbonFile[] getFiles(FileStatus[] listStatus) {
+ if (listStatus == null) {
+ return new CarbonFile[0];
+ }
+ CarbonFile[] files = new CarbonFile[listStatus.length];
+ for (int i = 0; i < files.length; i++) {
+ files[i] = new ViewFSCarbonFile(listStatus[i]);
+ }
+ return files;
+ }
+
+ @Override
+ public CarbonFile[] listFiles() {
+ FileStatus[] listStatus = null;
+ try {
+ if (null != fileStatus && fileStatus.isDirectory()) {
+ Path path = fileStatus.getPath();
+ listStatus = path.getFileSystem(FileFactory.getConfiguration()).listStatus(path);
+ } else {
+ return null;
+ }
+ } catch (IOException ex) {
+ LOGGER.error("Exception occured" + ex.getMessage());
+ return new CarbonFile[0];
+ }
+ return getFiles(listStatus);
+ }
+
+ @Override
+ public CarbonFile[] listFiles(final CarbonFileFilter fileFilter) {
+ CarbonFile[] files = listFiles();
+ if (files != null && files.length >= 1) {
+ List<CarbonFile> fileList = new ArrayList<CarbonFile>(files.length);
+ for (int i = 0; i < files.length; i++) {
+ if (fileFilter.accept(files[i])) {
+ fileList.add(files[i]);
+ }
+ }
+ if (fileList.size() >= 1) {
+ return fileList.toArray(new CarbonFile[fileList.size()]);
+ } else {
+ return new CarbonFile[0];
+ }
+ }
+ return files;
+ }
+
+ @Override public CarbonFile getParentFile() {
+ Path parent = fileStatus.getPath().getParent();
+ return null == parent ? null : new ViewFSCarbonFile(parent);
+ }
+
+ @Override
+ public boolean renameForce(String changetoName) {
+ FileSystem fs;
+ try {
+ fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
+ if (fs instanceof ViewFileSystem) {
+ fs.delete(new Path(changetoName), true);
+ fs.rename(fileStatus.getPath(), new Path(changetoName));
+ return true;
+ } else {
+ return false;
+ }
+ } catch (IOException e) {
+ LOGGER.error("Exception occured" + e.getMessage());
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/impl/CompressedDataMeasureDataWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/CompressedDataMeasureDataWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/CompressedDataMeasureDataWrapper.java
new file mode 100644
index 0000000..fe754b1
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/CompressedDataMeasureDataWrapper.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.impl;
+
+import org.apache.carbondata.core.datastore.MeasureDataWrapper;
+import org.apache.carbondata.core.datastore.dataholder.CarbonReadDataHolder;
+
+public class CompressedDataMeasureDataWrapper implements MeasureDataWrapper {
+
+ private final CarbonReadDataHolder[] values;
+
+ public CompressedDataMeasureDataWrapper(final CarbonReadDataHolder[] values) {
+ this.values = values;
+ }
+
+ @Override public CarbonReadDataHolder[] getValues() {
+ return values;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java
new file mode 100644
index 0000000..d97cdf3
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/DFSFileHolderImpl.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.datastore.impl;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.FileHolder;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+
+public class DFSFileHolderImpl implements FileHolder {
+ /**
+ * cache to hold filename and its stream
+ */
+ private Map<String, FSDataInputStream> fileNameAndStreamCache;
+
+ public DFSFileHolderImpl() {
+ this.fileNameAndStreamCache =
+ new HashMap<String, FSDataInputStream>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ }
+
+ @Override public byte[] readByteArray(String filePath, long offset, int length)
+ throws IOException {
+ FSDataInputStream fileChannel = updateCache(filePath);
+ return read(fileChannel, length, offset);
+ }
+
+ /**
+ * This method will be used to check whether stream is already present in
+ * cache or not for filepath if not present then create it and then add to
+ * cache, other wise get from cache
+ *
+ * @param filePath fully qualified file path
+ * @return channel
+ */
+ private FSDataInputStream updateCache(String filePath) throws IOException {
+ FSDataInputStream fileChannel = fileNameAndStreamCache.get(filePath);
+ if (null == fileChannel) {
+ Path pt = new Path(filePath);
+ FileSystem fs = FileSystem.get(FileFactory.getConfiguration());
+ fileChannel = fs.open(pt);
+ fileNameAndStreamCache.put(filePath, fileChannel);
+ }
+ return fileChannel;
+ }
+
+ /**
+ * This method will be used to read from file based on number of bytes to be read and positon
+ *
+ * @param channel file channel
+ * @param size number of bytes
+ * @param offset position
+ * @return byte buffer
+ */
+ private byte[] read(FSDataInputStream channel, int size, long offset) throws IOException {
+ byte[] byteBffer = new byte[size];
+ channel.seek(offset);
+ channel.readFully(byteBffer);
+ return byteBffer;
+ }
+
+ /**
+ * This method will be used to read from file based on number of bytes to be read and positon
+ *
+ * @param channel file channel
+ * @param size number of bytes
+ * @return byte buffer
+ */
+ private byte[] read(FSDataInputStream channel, int size) throws IOException {
+ byte[] byteBffer = new byte[size];
+ channel.readFully(byteBffer);
+ return byteBffer;
+ }
+
+ @Override public int readInt(String filePath, long offset) throws IOException {
+ FSDataInputStream fileChannel = updateCache(filePath);
+ fileChannel.seek(offset);
+ return fileChannel.readInt();
+ }
+
+ @Override public long readDouble(String filePath, long offset) throws IOException {
+ FSDataInputStream fileChannel = updateCache(filePath);
+ fileChannel.seek(offset);
+ return fileChannel.readLong();
+ }
+
+ @Override public void finish() throws IOException {
+ for (Entry<String, FSDataInputStream> entry : fileNameAndStreamCache.entrySet()) {
+ FSDataInputStream channel = entry.getValue();
+ if (null != channel) {
+ channel.close();
+ }
+ }
+ }
+
+ @Override public byte[] readByteArray(String filePath, int length) throws IOException {
+ FSDataInputStream fileChannel = updateCache(filePath);
+ return read(fileChannel, length);
+ }
+
+ @Override public long readLong(String filePath, long offset) throws IOException {
+ FSDataInputStream fileChannel = updateCache(filePath);
+ fileChannel.seek(offset);
+ return fileChannel.readLong();
+ }
+
+ @Override public int readInt(String filePath) throws IOException {
+ FSDataInputStream fileChannel = updateCache(filePath);
+ return fileChannel.readInt();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
new file mode 100644
index 0000000..db108ed
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
@@ -0,0 +1,485 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.impl;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.zip.GZIPInputStream;
+
+import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.filesystem.AlluxioCarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.HDFSCarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.LocalCarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.ViewFSCarbonFile;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.compress.BZip2Codec;
+import org.apache.hadoop.io.compress.GzipCodec;
+
+public final class FileFactory {
+ private static Configuration configuration = null;
+
+ static {
+ configuration = new Configuration();
+ configuration.addResource(new Path("../core-default.xml"));
+ }
+
+ private FileFactory() {
+
+ }
+
+ public static Configuration getConfiguration() {
+ return configuration;
+ }
+
+ public static FileHolder getFileHolder(FileType fileType) {
+ switch (fileType) {
+ case LOCAL:
+ return new FileHolderImpl();
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ return new DFSFileHolderImpl();
+ default:
+ return new FileHolderImpl();
+ }
+ }
+
+ public static FileType getFileType(String path) {
+ if (path.startsWith(CarbonUtil.HDFS_PREFIX)) {
+ return FileType.HDFS;
+ }
+ else if (path.startsWith(CarbonUtil.ALLUXIO_PREFIX)) {
+ return FileType.ALLUXIO;
+ }
+ else if (path.startsWith(CarbonUtil.VIEWFS_PREFIX)) {
+ return FileType.VIEWFS;
+ }
+ return FileType.LOCAL;
+ }
+
+ public static CarbonFile getCarbonFile(String path, FileType fileType) {
+ switch (fileType) {
+ case LOCAL:
+ return new LocalCarbonFile(path);
+ case HDFS:
+ return new HDFSCarbonFile(path);
+ case ALLUXIO:
+ return new AlluxioCarbonFile(path);
+ case VIEWFS:
+ return new ViewFSCarbonFile(path);
+ default:
+ return new LocalCarbonFile(path);
+ }
+ }
+
+ public static DataInputStream getDataInputStream(String path, FileType fileType)
+ throws IOException {
+ return getDataInputStream(path, fileType, -1);
+ }
+
+ public static DataInputStream getDataInputStream(String path, FileType fileType, int bufferSize)
+ throws IOException {
+ path = path.replace("\\", "/");
+ boolean gzip = path.endsWith(".gz");
+ boolean bzip2 = path.endsWith(".bz2");
+ InputStream stream;
+ switch (fileType) {
+ case LOCAL:
+ if (gzip) {
+ stream = new GZIPInputStream(new FileInputStream(path));
+ } else if (bzip2) {
+ stream = new BZip2CompressorInputStream(new FileInputStream(path));
+ } else {
+ stream = new FileInputStream(path);
+ }
+ break;
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ Path pt = new Path(path);
+ FileSystem fs = pt.getFileSystem(configuration);
+ if (bufferSize == -1) {
+ stream = fs.open(pt);
+ } else {
+ stream = fs.open(pt, bufferSize);
+ }
+ if (gzip) {
+ GzipCodec codec = new GzipCodec();
+ stream = codec.createInputStream(stream);
+ } else if (bzip2) {
+ BZip2Codec codec = new BZip2Codec();
+ stream = codec.createInputStream(stream);
+ }
+ break;
+ default:
+ throw new UnsupportedOperationException("unsupported file system");
+ }
+ return new DataInputStream(new BufferedInputStream(stream));
+ }
+
+ /**
+ * return the datainputStream which is seek to the offset of file
+ *
+ * @param path
+ * @param fileType
+ * @param bufferSize
+ * @param offset
+ * @return DataInputStream
+ * @throws IOException
+ */
+ public static DataInputStream getDataInputStream(String path, FileType fileType, int bufferSize,
+ long offset) throws IOException {
+ path = path.replace("\\", "/");
+ switch (fileType) {
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ Path pt = new Path(path);
+ FileSystem fs = pt.getFileSystem(configuration);
+ FSDataInputStream stream = fs.open(pt, bufferSize);
+ stream.seek(offset);
+ return new DataInputStream(new BufferedInputStream(stream));
+ default:
+ FileInputStream fis = new FileInputStream(path);
+ long actualSkipSize = 0;
+ long skipSize = offset;
+ while (actualSkipSize != offset) {
+ actualSkipSize += fis.skip(skipSize);
+ skipSize = skipSize - actualSkipSize;
+ }
+ return new DataInputStream(new BufferedInputStream(fis));
+ }
+ }
+
+ public static DataOutputStream getDataOutputStream(String path, FileType fileType)
+ throws IOException {
+ path = path.replace("\\", "/");
+ switch (fileType) {
+ case LOCAL:
+ return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path)));
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ Path pt = new Path(path);
+ FileSystem fs = pt.getFileSystem(configuration);
+ FSDataOutputStream stream = fs.create(pt, true);
+ return stream;
+ default:
+ return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path)));
+ }
+ }
+
+ public static DataOutputStream getDataOutputStream(String path, FileType fileType, int bufferSize,
+ boolean append) throws IOException {
+ path = path.replace("\\", "/");
+ switch (fileType) {
+ case LOCAL:
+ return new DataOutputStream(
+ new BufferedOutputStream(new FileOutputStream(path, append), bufferSize));
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ Path pt = new Path(path);
+ FileSystem fs = pt.getFileSystem(configuration);
+ FSDataOutputStream stream = null;
+ if (append) {
+ // append to a file only if file already exists else file not found
+ // exception will be thrown by hdfs
+ if (CarbonUtil.isFileExists(path)) {
+ stream = fs.append(pt, bufferSize);
+ } else {
+ stream = fs.create(pt, true, bufferSize);
+ }
+ } else {
+ stream = fs.create(pt, true, bufferSize);
+ }
+ return stream;
+ default:
+ return new DataOutputStream(
+ new BufferedOutputStream(new FileOutputStream(path), bufferSize));
+ }
+ }
+
+ public static DataOutputStream getDataOutputStream(String path, FileType fileType, int bufferSize,
+ long blockSize) throws IOException {
+ path = path.replace("\\", "/");
+ switch (fileType) {
+ case LOCAL:
+ return new DataOutputStream(
+ new BufferedOutputStream(new FileOutputStream(path), bufferSize));
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ Path pt = new Path(path);
+ FileSystem fs = pt.getFileSystem(configuration);
+ FSDataOutputStream stream =
+ fs.create(pt, true, bufferSize, fs.getDefaultReplication(pt), blockSize);
+ return stream;
+ default:
+ return new DataOutputStream(
+ new BufferedOutputStream(new FileOutputStream(path), bufferSize));
+ }
+ }
+
+ /**
+ * This method checks the given path exists or not and also is it file or
+ * not if the performFileCheck is true
+ *
+ * @param filePath - Path
+ * @param fileType - FileType Local/HDFS
+ * @param performFileCheck - Provide false for folders, true for files and
+ */
+ public static boolean isFileExist(String filePath, FileType fileType, boolean performFileCheck)
+ throws IOException {
+ filePath = filePath.replace("\\", "/");
+ switch (fileType) {
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ Path path = new Path(filePath);
+ FileSystem fs = path.getFileSystem(configuration);
+ if (performFileCheck) {
+ return fs.exists(path) && fs.isFile(path);
+ } else {
+ return fs.exists(path);
+ }
+
+ case LOCAL:
+ default:
+ File defaultFile = new File(filePath);
+
+ if (performFileCheck) {
+ return defaultFile.exists() && defaultFile.isFile();
+ } else {
+ return defaultFile.exists();
+ }
+ }
+ }
+
+ /**
+ * This method checks the given path exists or not and also is it file or
+ * not if the performFileCheck is true
+ *
+ * @param filePath - Path
+ * @param fileType - FileType Local/HDFS
+ */
+ public static boolean isFileExist(String filePath, FileType fileType) throws IOException {
+ filePath = filePath.replace("\\", "/");
+ switch (fileType) {
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ Path path = new Path(filePath);
+ FileSystem fs = path.getFileSystem(configuration);
+ return fs.exists(path);
+
+ case LOCAL:
+ default:
+ File defaultFile = new File(filePath);
+ return defaultFile.exists();
+ }
+ }
+
+ public static boolean createNewFile(String filePath, FileType fileType) throws IOException {
+ filePath = filePath.replace("\\", "/");
+ switch (fileType) {
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ Path path = new Path(filePath);
+ FileSystem fs = path.getFileSystem(configuration);
+ return fs.createNewFile(path);
+
+ case LOCAL:
+ default:
+ File file = new File(filePath);
+ return file.createNewFile();
+ }
+ }
+
+ public static boolean deleteFile(String filePath, FileType fileType) throws IOException {
+ filePath = filePath.replace("\\", "/");
+ switch (fileType) {
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ Path path = new Path(filePath);
+ FileSystem fs = path.getFileSystem(configuration);
+ return fs.delete(path, true);
+
+ case LOCAL:
+ default:
+ File file = new File(filePath);
+ return deleteAllFilesOfDir(file);
+ }
+ }
+
+ public static boolean deleteAllFilesOfDir(File path) {
+ if (!path.exists()) {
+ return true;
+ }
+ if (path.isFile()) {
+ return path.delete();
+ }
+ File[] files = path.listFiles();
+ for (int i = 0; i < files.length; i++) {
+ deleteAllFilesOfDir(files[i]);
+ }
+ return path.delete();
+ }
+
+ public static boolean mkdirs(String filePath, FileType fileType) throws IOException {
+ filePath = filePath.replace("\\", "/");
+ switch (fileType) {
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ Path path = new Path(filePath);
+ FileSystem fs = path.getFileSystem(configuration);
+ return fs.mkdirs(path);
+ case LOCAL:
+ default:
+ File file = new File(filePath);
+ return file.mkdirs();
+ }
+ }
+
+ /**
+ * for getting the dataoutput stream using the hdfs filesystem append API.
+ *
+ * @param path
+ * @param fileType
+ * @return
+ * @throws IOException
+ */
+ public static DataOutputStream getDataOutputStreamUsingAppend(String path, FileType fileType)
+ throws IOException {
+ path = path.replace("\\", "/");
+ switch (fileType) {
+ case LOCAL:
+ return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path, true)));
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ Path pt = new Path(path);
+ FileSystem fs = pt.getFileSystem(configuration);
+ FSDataOutputStream stream = fs.append(pt);
+ return stream;
+ default:
+ return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path)));
+ }
+ }
+
+ /**
+ * for creating a new Lock file and if it is successfully created
+ * then in case of abrupt shutdown then the stream to that file will be closed.
+ *
+ * @param filePath
+ * @param fileType
+ * @return
+ * @throws IOException
+ */
+ public static boolean createNewLockFile(String filePath, FileType fileType) throws IOException {
+ filePath = filePath.replace("\\", "/");
+ switch (fileType) {
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ Path path = new Path(filePath);
+ FileSystem fs = path.getFileSystem(configuration);
+ if (fs.createNewFile(path)) {
+ fs.deleteOnExit(path);
+ return true;
+ }
+ return false;
+ case LOCAL:
+ default:
+ File file = new File(filePath);
+ return file.createNewFile();
+ }
+ }
+
+ public enum FileType {
+ LOCAL, HDFS, ALLUXIO, VIEWFS
+ }
+
+ /**
+ * below method will be used to update the file path
+ * for local type
+ * it removes the file:/ from the path
+ *
+ * @param filePath
+ * @return updated file path without url for local
+ */
+ public static String getUpdatedFilePath(String filePath) {
+ FileType fileType = getFileType(filePath);
+ switch (fileType) {
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ return filePath;
+ case LOCAL:
+ default:
+ Path pathWithoutSchemeAndAuthority =
+ Path.getPathWithoutSchemeAndAuthority(new Path(filePath));
+ return pathWithoutSchemeAndAuthority.toString();
+ }
+ }
+
+ /**
+ * It computes size of directory
+ *
+ * @param filePath
+ * @return size in bytes
+ * @throws IOException
+ */
+ public static long getDirectorySize(String filePath) throws IOException {
+ FileType fileType = getFileType(filePath);
+ switch (fileType) {
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ Path path = new Path(filePath);
+ FileSystem fs = path.getFileSystem(configuration);
+ return fs.getContentSummary(path).getLength();
+ case LOCAL:
+ default:
+ File file = new File(filePath);
+ return FileUtils.sizeOfDirectory(file);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
new file mode 100644
index 0000000..48928c6
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.impl;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.FileHolder;
+
+public class FileHolderImpl implements FileHolder {
+ /**
+ * cache to hold filename and its stream
+ */
+ private Map<String, FileChannel> fileNameAndStreamCache;
+
+ /**
+ * FileHolderImpl Constructor
+ * It will create the cache
+ */
+ public FileHolderImpl() {
+ this.fileNameAndStreamCache =
+ new HashMap<String, FileChannel>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ }
+
+ public FileHolderImpl(int capacity) {
+ this.fileNameAndStreamCache = new HashMap<String, FileChannel>(capacity);
+ }
+
+ /**
+ * This method will be used to read the byte array from file based on offset
+ * and length(number of bytes) need to read
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @param length number of bytes to be read
+ * @return read byte array
+ */
+ @Override public byte[] readByteArray(String filePath, long offset, int length)
+ throws IOException {
+ FileChannel fileChannel = updateCache(filePath);
+ ByteBuffer byteBffer = read(fileChannel, length, offset);
+ return byteBffer.array();
+ }
+
+ /**
+ * This method will be used to close all the streams currently present in the cache
+ */
+ @Override public void finish() throws IOException {
+ for (Entry<String, FileChannel> entry : fileNameAndStreamCache.entrySet()) {
+ FileChannel channel = entry.getValue();
+ if (null != channel) {
+ channel.close();
+ }
+ }
+ }
+
+ /**
+ * This method will be used to read int from file from postion(offset), here
+ * length will be always 4 bacause int byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read int
+ */
+ @Override public int readInt(String filePath, long offset) throws IOException {
+ FileChannel fileChannel = updateCache(filePath);
+ ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.INT_SIZE_IN_BYTE, offset);
+ return byteBffer.getInt();
+ }
+
+ /**
+ * This method will be used to read int from file from postion(offset), here
+ * length will be always 4 bacause int byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @return read int
+ */
+ @Override public int readInt(String filePath) throws IOException {
+ FileChannel fileChannel = updateCache(filePath);
+ ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.INT_SIZE_IN_BYTE);
+ return byteBffer.getInt();
+ }
+
+ /**
+ * This method will be used to read int from file from postion(offset), here
+ * length will be always 4 bacause int byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read int
+ */
+ @Override public long readDouble(String filePath, long offset) throws IOException {
+ FileChannel fileChannel = updateCache(filePath);
+ ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.LONG_SIZE_IN_BYTE, offset);
+ return byteBffer.getLong();
+ }
+
+ /**
+ * This method will be used to check whether stream is already present in
+ * cache or not for filepath if not present then create it and then add to
+ * cache, other wise get from cache
+ *
+ * @param filePath fully qualified file path
+ * @return channel
+ */
+ private FileChannel updateCache(String filePath) throws FileNotFoundException {
+ FileChannel fileChannel = fileNameAndStreamCache.get(filePath);
+ if (null == fileChannel) {
+ FileInputStream stream = new FileInputStream(filePath);
+ fileChannel = stream.getChannel();
+ fileNameAndStreamCache.put(filePath, fileChannel);
+ }
+ return fileChannel;
+ }
+
+ /**
+ * This method will be used to read from file based on number of bytes to be read and positon
+ *
+ * @param channel file channel
+ * @param size number of bytes
+ * @param offset position
+ * @return byte buffer
+ */
+ private ByteBuffer read(FileChannel channel, int size, long offset) throws IOException {
+ ByteBuffer byteBffer = ByteBuffer.allocate(size);
+ channel.position(offset);
+ channel.read(byteBffer);
+ byteBffer.rewind();
+ return byteBffer;
+ }
+
+ /**
+ * This method will be used to read from file based on number of bytes to be read and positon
+ *
+ * @param channel file channel
+ * @param size number of bytes
+ * @return byte buffer
+ */
+ private ByteBuffer read(FileChannel channel, int size) throws IOException {
+ ByteBuffer byteBffer = ByteBuffer.allocate(size);
+ channel.read(byteBffer);
+ byteBffer.rewind();
+ return byteBffer;
+ }
+
+
+ /**
+ * This method will be used to read the byte array from file based on length(number of bytes)
+ *
+ * @param filePath fully qualified file path
+ * @param length number of bytes to be read
+ * @return read byte array
+ */
+ @Override public byte[] readByteArray(String filePath, int length) throws IOException {
+ FileChannel fileChannel = updateCache(filePath);
+ ByteBuffer byteBffer = read(fileChannel, length);
+ return byteBffer.array();
+ }
+
+ /**
+ * This method will be used to read long from file from postion(offset), here
+ * length will be always 8 bacause int byte size is 8
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read long
+ */
+ @Override public long readLong(String filePath, long offset) throws IOException {
+ FileChannel fileChannel = updateCache(filePath);
+ ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.LONG_SIZE_IN_BYTE, offset);
+ return byteBffer.getLong();
+ }
+
+}
[51/57] [abbrv] incubator-carbondata git commit: fix style
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneLong.java
deleted file mode 100644
index c22464d..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneLong.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.compression.none;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNoneLong extends ValueCompressionHolder<long[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNoneLong.class.getName());
- /**
- * longCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
- /**
- * value.
- */
- protected long[] value;
-
- private DataType actualDataType;
-
- private MeasureDataChunkStore<long[]> measureChunkStore;
-
- public CompressionNoneLong(DataType actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- @Override public void setValue(long[] value) { this.value = value; }
-
- @Override public long[] getValue() { return this.value; }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_LONG, value);
- }
-
- @Override
- public void uncompress(DataType dataType, byte[] data, int offset, int length,
- int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, data, offset, length);
- setUncompressedValues(value);
- }
-
- @Override public void setValueInBytes(byte[] byteValue) {
- ByteBuffer buffer = ByteBuffer.wrap(byteValue);
- this.value = ValueCompressionUtil.convertToLongArray(buffer, byteValue.length);
- }
-
- @Override public long getLongValue(int index) {
- return measureChunkStore.getLong(index);
- }
-
- @Override public double getDoubleValue(int index) {
- return measureChunkStore.getLong(index);
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException("Get big decimal is not supported");
- }
-
- private void setUncompressedValues(long[] data) {
- this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_LONG, data.length);
- this.measureChunkStore.putData(data);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneShort.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneShort.java
deleted file mode 100644
index f0b1c99..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneShort.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.compression.none;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNoneShort extends ValueCompressionHolder<short[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNoneShort.class.getName());
-
- /**
- * shortCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- /**
- * value.
- */
- private short[] shortValue;
-
- private MeasureDataChunkStore<short[]> measureChunkStore;
-
- private DataType actualDataType;
-
- public CompressionNoneShort(DataType actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- @Override public void setValue(short[] shortValue) {
- this.shortValue = shortValue;
- }
-
- @Override public short[] getValue() { return this.shortValue; }
-
- @Override
- public void uncompress(DataType dataType, byte[] data, int offset, int length,
- int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, data, offset, length);
- setUncompressedValues(shortValue);
- }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_SHORT, shortValue);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- shortValue = ValueCompressionUtil.convertToShortArray(buffer, value.length);
- }
-
- @Override public long getLongValue(int index) {
- return measureChunkStore.getShort(index);
- }
-
- @Override public double getDoubleValue(int index) {
- return measureChunkStore.getShort(index);
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal is not defined for CompressionNonShort");
- }
-
- private void setUncompressedValues(short[] data) {
- this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_SHORT, data.length);
- this.measureChunkStore.putData(data);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/type/CompressionBigDecimal.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/type/CompressionBigDecimal.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/type/CompressionBigDecimal.java
deleted file mode 100644
index 8233e90..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/type/CompressionBigDecimal.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.datastorage.compression.type;
-
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.BigDecimalCompressionFinder;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-/**
- * Big decimal compression
- */
-public class CompressionBigDecimal<T> extends ValueCompressionHolder<T> {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionBigDecimal.class.getName());
-
- private BigDecimalCompressionFinder compressionFinder;
-
- /**
- * leftPart before decimal
- */
- private ValueCompressionHolder leftPart;
-
- /**
- * rightPart after decimal
- */
- private ValueCompressionHolder rightPart;
-
- private double divisionFactor;
-
- private boolean isDecimalPlacesNotZero;
-
- public CompressionBigDecimal(BigDecimalCompressionFinder compressionFinder,
- ValueCompressionHolder leftPart, ValueCompressionHolder rightPart) {
- this.compressionFinder = compressionFinder;
- this.leftPart = leftPart;
- this.rightPart = rightPart;
- }
-
- @Override public void setValue(T value) {
- Object[] values = (Object[]) value;
- leftPart.setValue(values[0]);
- rightPart.setValue(values[1]);
- }
-
- @Override
- public void uncompress(DataType dataType, byte[] data, int offset, int length,
- int decimalPlaces, Object maxValueObject) {
- if (decimalPlaces > 0) {
- this.isDecimalPlacesNotZero = true;
- }
- this.divisionFactor = Math.pow(10, decimalPlaces);
-
- ByteBuffer buffer = ByteBuffer.wrap(data, offset, length);
- int leftPathLength = buffer.getInt();
- int rightPartLength = length - leftPathLength - CarbonCommonConstants.INT_SIZE_IN_BYTE;
- Long[] maxValue = (Long[]) maxValueObject;
- leftPart.uncompress(compressionFinder.getLeftConvertedDataType(), data,
- offset + CarbonCommonConstants.INT_SIZE_IN_BYTE, leftPathLength, decimalPlaces,
- maxValue[0]);
- rightPart.uncompress(compressionFinder.getRightConvertedDataType(), data,
- offset + CarbonCommonConstants.INT_SIZE_IN_BYTE + leftPathLength, rightPartLength,
- decimalPlaces, maxValue[1]);
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long is not defined for CompressionBigDecimal");
- }
-
- @Override public double getDoubleValue(int index) {
- throw new UnsupportedOperationException(
- "Double is not defined for CompressionBigDecimal");
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- long leftValue = leftPart.getLongValue(index);
- long rightValue = 0;
- if (isDecimalPlacesNotZero) {
- rightValue = rightPart.getLongValue(index);
- }
- String decimalPart = Double.toString(rightValue / this.divisionFactor);
- String bigdStr = Long.toString(leftValue) + CarbonCommonConstants.POINT + decimalPart
- .substring(decimalPart.indexOf(".") + 1, decimalPart.length());
- return new BigDecimal(bigdStr);
- }
-
- @Override public T getValue() {
- Object[] values = new Object[2];
- values[0] = leftPart;
- values[1] = rightPart;
- return (T) values;
- }
-
- @Override public void setValueInBytes(byte[] value) {
- LOGGER.error("setValueInBytes() is not defined for CompressionBigDecimal");
- }
-
- @Override public void compress() {
- leftPart.compress();
- rightPart.compress();
- }
-
- @Override
- public byte[] getCompressedData() {
- byte[] leftdata = leftPart.getCompressedData();
- byte[] rightdata = rightPart.getCompressedData();
- ByteBuffer byteBuffer = ByteBuffer
- .allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE + leftdata.length
- + rightdata.length);
- byteBuffer.putInt(leftdata.length);
- byteBuffer.put(leftdata);
- byteBuffer.put(rightdata);
- byteBuffer.flip();
- return byteBuffer.array();
- }
-
- @Override public void freeMemory() {
- leftPart.freeMemory();
- rightPart.freeMemory();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/dataholder/CarbonReadDataHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/dataholder/CarbonReadDataHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/dataholder/CarbonReadDataHolder.java
deleted file mode 100644
index 29ff254..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/dataholder/CarbonReadDataHolder.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.dataholder;
-
-import java.math.BigDecimal;
-
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-
-// This class is used with Uncompressor to hold the decompressed column chunk in memory
-public class CarbonReadDataHolder {
-
- private ValueCompressionHolder unCompressValue;
-
- public CarbonReadDataHolder(ValueCompressionHolder unCompressValue) {
- this.unCompressValue = unCompressValue;
- }
-
- public long getReadableLongValueByIndex(int index) {
- return this.unCompressValue.getLongValue(index);
- }
-
- public BigDecimal getReadableBigDecimalValueByIndex(int index) {
- return this.unCompressValue.getBigDecimalValue(index);
- }
-
- public double getReadableDoubleValueByIndex(int index) {
- return this.unCompressValue.getDoubleValue(index);
- }
-
- public void freeMemory() {
- unCompressValue.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/dataholder/CarbonWriteDataHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/dataholder/CarbonWriteDataHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/dataholder/CarbonWriteDataHolder.java
deleted file mode 100644
index e6d9123..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/dataholder/CarbonWriteDataHolder.java
+++ /dev/null
@@ -1,280 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.dataholder;
-
-public class CarbonWriteDataHolder {
- /**
- * doubleValues
- */
- private double[] doubleValues;
-
- /**
- * longValues
- */
- private long[] longValues;
-
- /**
- * bigDecimal left part
- */
- private long[] bigDecimalLeftValues;
-
- /**
- * bigDecimal right part
- */
- private long[] bigDecimalRightValues;
- /**
- * byteValues
- */
- private byte[][] byteValues;
-
- /**
- * byteValues for no dictionary and non kettle flow.
- */
- private byte[][][] byteValuesForNonDictionary;
-
- /**
- * byteValues
- */
- private byte[][][] columnByteValues;
-
- /**
- * size
- */
- private int size;
-
- /**
- * totalSize
- */
- private int totalSize;
-
- /**
- * Method to initialise double array
- *
- * @param size
- */
- public void initialiseDoubleValues(int size) {
- if (size < 1) {
- throw new IllegalArgumentException("Invalid array size");
- }
- doubleValues = new double[size];
- }
-
- public void reset() {
- size = 0;
- totalSize = 0;
- }
-
- /**
- * Method to initialise double array
- * TODO Remove after kettle flow got removed.
- *
- * @param size
- */
- public void initialiseByteArrayValues(int size) {
- if (size < 1) {
- throw new IllegalArgumentException("Invalid array size");
- }
-
- byteValues = new byte[size][];
- columnByteValues = new byte[size][][];
- }
-
- /**
- * Method to initialise byte array
- *
- * @param size
- */
- public void initialiseByteArrayValuesWithOutKettle(int size) {
- if (size < 1) {
- throw new IllegalArgumentException("Invalid array size");
- }
-
- byteValues = new byte[size][];
- }
-
- public void initialiseByteArrayValuesForNonDictionary(int size) {
- if (size < 1) {
- throw new IllegalArgumentException("Invalid array size");
- }
-
- byteValuesForNonDictionary = new byte[size][][];
- }
-
- /**
- * Method to initialise long array
- *
- * @param size
- */
- public void initialiseLongValues(int size) {
- if (size < 1) {
- throw new IllegalArgumentException("Invalid array size");
- }
- longValues = new long[size];
- }
-
- public void initialiseBigDecimalValues(int size) {
- if (size < 1) {
- throw new IllegalArgumentException("Invalid array size");
- }
- bigDecimalLeftValues = new long[size];
- bigDecimalRightValues = new long[size];
- }
-
- /**
- * set double value by index
- *
- * @param index
- * @param value
- */
- public void setWritableDoubleValueByIndex(int index, Object value) {
- doubleValues[index] = (Double) value;
- size++;
- }
-
- /**
- * set double value by index
- *
- * @param index
- * @param value
- */
- public void setWritableLongValueByIndex(int index, Object value) {
- longValues[index] = (Long) value;
- size++;
- }
-
- /**
- * set bigdecimal value by index
- *
- * @param index
- * @param value
- */
- public void setWritableBigDecimalValueByIndex(int index, long[] value) {
- bigDecimalLeftValues[index] = value[0];
- bigDecimalRightValues[index] = value[1];
- size++;
- }
- /**
- * set byte array value by index
- *
- * @param index
- * @param value
- */
- public void setWritableByteArrayValueByIndex(int index, byte[] value) {
- byteValues[index] = value;
- size++;
- if (null != value) totalSize += value.length;
- }
-
- public void setWritableNonDictByteArrayValueByIndex(int index, byte[][] value) {
- byteValuesForNonDictionary[index] = value;
- size++;
- if (null != value) totalSize += value.length;
- }
-
- /**
- * set byte array value by index
- */
- public void setWritableByteArrayValueByIndex(int index, int mdKeyIndex, Object[] columnData) {
- int l = 0;
- columnByteValues[index] = new byte[columnData.length - (mdKeyIndex + 1)][];
- for (int i = mdKeyIndex + 1; i < columnData.length; i++) {
- columnByteValues[index][l++] = (byte[]) columnData[i];
- }
- }
-
- /**
- * Get Writable Double Values
- */
- public double[] getWritableDoubleValues() {
- if (size < doubleValues.length) {
- double[] temp = new double[size];
- System.arraycopy(doubleValues, 0, temp, 0, size);
- doubleValues = temp;
- }
- return doubleValues;
- }
-
- /**
- * Get writable byte array values
- */
- public byte[] getWritableByteArrayValues() {
- byte[] temp = new byte[totalSize];
- int startIndexToCopy = 0;
- for (int i = 0; i < size; i++) {
- System.arraycopy(byteValues[i], 0, temp, startIndexToCopy, byteValues[i].length);
- startIndexToCopy += byteValues[i].length;
- }
- return temp;
- }
-
- public byte[][] getByteArrayValues() {
- if (size < byteValues.length) {
- byte[][] temp = new byte[size][];
- System.arraycopy(byteValues, 0, temp, 0, size);
- byteValues = temp;
- }
- return byteValues;
- }
-
- public byte[][][] getNonDictByteArrayValues() {
- if (size < byteValuesForNonDictionary.length) {
- byte[][][] temp = new byte[size][][];
- System.arraycopy(byteValuesForNonDictionary, 0, temp, 0, size);
- byteValuesForNonDictionary = temp;
- }
- return byteValuesForNonDictionary;
- }
-
- /**
- * Get Writable Double Values
- *
- * @return
- */
- public long[] getWritableLongValues() {
- if (size < longValues.length) {
- long[] temp = new long[size];
- System.arraycopy(longValues, 0, temp, 0, size);
- longValues = temp;
- }
- return longValues;
- }
-
- /**
- * Get Writable bigdecimal Values
- *
- * @return
- */
- public long[][] getWritableBigDecimalValues() {
- long[][] bigDecimalValues = new long[2][];
- if (size < bigDecimalLeftValues.length) {
- long[] temp = new long[size];
- System.arraycopy(bigDecimalLeftValues, 0, temp, 0, size);
- bigDecimalLeftValues = temp;
- }
- if (size < bigDecimalRightValues.length) {
- long[] temp = new long[size];
- System.arraycopy(bigDecimalRightValues, 0, temp, 0, size);
- bigDecimalRightValues = temp;
- }
- bigDecimalValues[0]= bigDecimalLeftValues;
- bigDecimalValues[1] = bigDecimalRightValues;
- return bigDecimalValues;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/AbstractDFSCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/AbstractDFSCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/AbstractDFSCarbonFile.java
deleted file mode 100644
index 8963702..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/AbstractDFSCarbonFile.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.filesystem;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
-import org.apache.carbondata.core.util.CarbonUtil;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-public abstract class AbstractDFSCarbonFile implements CarbonFile {
- /**
- * LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(AbstractDFSCarbonFile.class.getName());
- protected FileStatus fileStatus;
- protected FileSystem fs;
-
- public AbstractDFSCarbonFile(String filePath) {
- filePath = filePath.replace("\\", "/");
- Path path = new Path(filePath);
- try {
- fs = path.getFileSystem(FileFactory.getConfiguration());
- fileStatus = fs.getFileStatus(path);
- } catch (IOException e) {
- LOGGER.error("Exception occured:" + e.getMessage());
- }
- }
-
- public AbstractDFSCarbonFile(Path path) {
- try {
- fs = path.getFileSystem(FileFactory.getConfiguration());
- fileStatus = fs.getFileStatus(path);
- } catch (IOException e) {
- LOGGER.error("Exception occured:" + e.getMessage());
- }
- }
-
- public AbstractDFSCarbonFile(FileStatus fileStatus) {
- this.fileStatus = fileStatus;
- }
-
- @Override public boolean createNewFile() {
- Path path = fileStatus.getPath();
- try {
- return fs.createNewFile(path);
- } catch (IOException e) {
- return false;
- }
- }
-
- @Override public String getAbsolutePath() {
- return fileStatus.getPath().toString();
- }
-
- @Override public String getName() {
- return fileStatus.getPath().getName();
- }
-
- @Override public boolean isDirectory() {
- return fileStatus.isDirectory();
- }
-
- @Override public boolean exists() {
- try {
- if (null != fileStatus) {
- fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
- return fs.exists(fileStatus.getPath());
- }
- } catch (IOException e) {
- LOGGER.error("Exception occured:" + e.getMessage());
- }
- return false;
- }
-
- @Override public String getCanonicalPath() {
- return getAbsolutePath();
- }
-
- @Override public String getPath() {
- return getAbsolutePath();
- }
-
- @Override public long getSize() {
- return fileStatus.getLen();
- }
-
- public boolean renameTo(String changetoName) {
- FileSystem fs;
- try {
- fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
- return fs.rename(fileStatus.getPath(), new Path(changetoName));
- } catch (IOException e) {
- LOGGER.error("Exception occured:" + e.getMessage());
- return false;
- }
- }
-
- public boolean delete() {
- FileSystem fs;
- try {
- fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
- return fs.delete(fileStatus.getPath(), true);
- } catch (IOException e) {
- LOGGER.error("Exception occured:" + e.getMessage());
- return false;
- }
- }
-
- @Override public long getLastModifiedTime() {
- return fileStatus.getModificationTime();
- }
-
- @Override public boolean setLastModifiedTime(long timestamp) {
- try {
- fs.setTimes(fileStatus.getPath(), timestamp, timestamp);
- } catch (IOException e) {
- return false;
- }
- return true;
- }
-
- /**
- * This method will delete the data in file data from a given offset
- */
- @Override public boolean truncate(String fileName, long validDataEndOffset) {
- DataOutputStream dataOutputStream = null;
- DataInputStream dataInputStream = null;
- boolean fileTruncatedSuccessfully = false;
- // if bytes to read less than 1024 then buffer size should be equal to the given offset
- int bufferSize = validDataEndOffset > CarbonCommonConstants.BYTE_TO_KB_CONVERSION_FACTOR ?
- CarbonCommonConstants.BYTE_TO_KB_CONVERSION_FACTOR :
- (int) validDataEndOffset;
- // temporary file name
- String tempWriteFilePath = fileName + CarbonCommonConstants.TEMPWRITEFILEEXTENSION;
- FileFactory.FileType fileType = FileFactory.getFileType(fileName);
- try {
- CarbonFile tempFile = null;
- // delete temporary file if it already exists at a given path
- if (FileFactory.isFileExist(tempWriteFilePath, fileType)) {
- tempFile = FileFactory.getCarbonFile(tempWriteFilePath, fileType);
- tempFile.delete();
- }
- // create new temporary file
- FileFactory.createNewFile(tempWriteFilePath, fileType);
- tempFile = FileFactory.getCarbonFile(tempWriteFilePath, fileType);
- byte[] buff = new byte[bufferSize];
- dataInputStream = FileFactory.getDataInputStream(fileName, fileType);
- // read the data
- int read = dataInputStream.read(buff, 0, buff.length);
- dataOutputStream = FileFactory.getDataOutputStream(tempWriteFilePath, fileType);
- dataOutputStream.write(buff, 0, read);
- long remaining = validDataEndOffset - read;
- // anytime we should not cross the offset to be read
- while (remaining > 0) {
- if (remaining > bufferSize) {
- buff = new byte[bufferSize];
- } else {
- buff = new byte[(int) remaining];
- }
- read = dataInputStream.read(buff, 0, buff.length);
- dataOutputStream.write(buff, 0, read);
- remaining = remaining - read;
- }
- CarbonUtil.closeStreams(dataInputStream, dataOutputStream);
- // rename the temp file to original file
- tempFile.renameForce(fileName);
- fileTruncatedSuccessfully = true;
- } catch (IOException e) {
- LOGGER.error("Exception occured while truncating the file " + e.getMessage());
- } finally {
- CarbonUtil.closeStreams(dataOutputStream, dataInputStream);
- }
- return fileTruncatedSuccessfully;
- }
-
- /**
- * This method will be used to check whether a file has been modified or not
- *
- * @param fileTimeStamp time to be compared with latest timestamp of file
- * @param endOffset file length to be compared with current length of file
- * @return
- */
- @Override public boolean isFileModified(long fileTimeStamp, long endOffset) {
- boolean isFileModified = false;
- if (getLastModifiedTime() > fileTimeStamp || getSize() > endOffset) {
- isFileModified = true;
- }
- return isFileModified;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/AlluxioCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/AlluxioCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/AlluxioCarbonFile.java
deleted file mode 100644
index e4ce2b6..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/AlluxioCarbonFile.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.filesystem;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-
-
-
-public class AlluxioCarbonFile extends AbstractDFSCarbonFile {
- /**
- * LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(AlluxioCarbonFile.class.getName());
-
- public AlluxioCarbonFile(String filePath) {
- super(filePath);
- }
-
- public AlluxioCarbonFile(Path path) {
- super(path);
- }
-
- public AlluxioCarbonFile(FileStatus fileStatus) {
- super(fileStatus);
- }
-
- /**
- * @param listStatus
- * @return
- */
- private CarbonFile[] getFiles(FileStatus[] listStatus) {
- if (listStatus == null) {
- return new CarbonFile[0];
- }
- CarbonFile[] files = new CarbonFile[listStatus.length];
- for (int i = 0; i < files.length; i++) {
- files[i] = new AlluxioCarbonFile(listStatus[i]);
- }
- return files;
- }
-
- @Override
- public CarbonFile[] listFiles() {
- FileStatus[] listStatus = null;
- try {
- if (null != fileStatus && fileStatus.isDirectory()) {
- Path path = fileStatus.getPath();
- listStatus = path.getFileSystem(FileFactory.getConfiguration()).listStatus(path);
- } else {
- return null;
- }
- } catch (IOException e) {
- LOGGER.error("Exception occured: " + e.getMessage());
- return new CarbonFile[0];
- }
- return getFiles(listStatus);
- }
-
- @Override
- public CarbonFile[] listFiles(final CarbonFileFilter fileFilter) {
- CarbonFile[] files = listFiles();
- if (files != null && files.length >= 1) {
- List<CarbonFile> fileList = new ArrayList<CarbonFile>(files.length);
- for (int i = 0; i < files.length; i++) {
- if (fileFilter.accept(files[i])) {
- fileList.add(files[i]);
- }
- }
- if (fileList.size() >= 1) {
- return fileList.toArray(new CarbonFile[fileList.size()]);
- } else {
- return new CarbonFile[0];
- }
- }
- return files;
- }
-
- @Override
- public CarbonFile getParentFile() {
- Path parent = fileStatus.getPath().getParent();
- return null == parent ? null : new AlluxioCarbonFile(parent);
- }
-
- @Override
- public boolean renameForce(String changetoName) {
- FileSystem fs;
- try {
- fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
- if (fs instanceof DistributedFileSystem) {
- ((DistributedFileSystem) fs).rename(fileStatus.getPath(), new Path(changetoName),
- org.apache.hadoop.fs.Options.Rename.OVERWRITE);
- return true;
- } else {
- return false;
- }
- } catch (IOException e) {
- LOGGER.error("Exception occured: " + e.getMessage());
- return false;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/CarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/CarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/CarbonFile.java
deleted file mode 100644
index 0ac4d52..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/CarbonFile.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.filesystem;
-
-public interface CarbonFile {
-
- String getAbsolutePath();
-
- CarbonFile[] listFiles(CarbonFileFilter fileFilter);
-
- CarbonFile[] listFiles();
-
- String getName();
-
- boolean isDirectory();
-
- boolean exists();
-
- String getCanonicalPath();
-
- CarbonFile getParentFile();
-
- String getPath();
-
- long getSize();
-
- boolean renameTo(String changetoName);
-
- boolean renameForce(String changetoName);
-
- boolean delete();
-
- boolean createNewFile();
-
- long getLastModifiedTime();
-
- boolean setLastModifiedTime(long timestamp);
-
- boolean truncate(String fileName, long validDataEndOffset);
-
- /**
- * This method will be used to check whether a file has been modified or not
- *
- * @param fileTimeStamp time to be compared with latest timestamp of file
- * @param endOffset file length to be compared with current length of file
- * @return
- */
- boolean isFileModified(long fileTimeStamp, long endOffset);
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/CarbonFileFilter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/CarbonFileFilter.java b/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/CarbonFileFilter.java
deleted file mode 100644
index e382f92..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/CarbonFileFilter.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.filesystem;
-
-public interface CarbonFileFilter {
- boolean accept(CarbonFile file);
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/HDFSCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/HDFSCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/HDFSCarbonFile.java
deleted file mode 100644
index a2aaca1..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/HDFSCarbonFile.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.filesystem;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-
-public class HDFSCarbonFile extends AbstractDFSCarbonFile {
- /**
- * LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(HDFSCarbonFile.class.getName());
-
- public HDFSCarbonFile(String filePath) {
- super(filePath);
- }
-
- public HDFSCarbonFile(Path path) {
- super(path);
- }
-
- public HDFSCarbonFile(FileStatus fileStatus) {
- super(fileStatus);
- }
-
- /**
- * @param listStatus
- * @return
- */
- private CarbonFile[] getFiles(FileStatus[] listStatus) {
- if (listStatus == null) {
- return new CarbonFile[0];
- }
- CarbonFile[] files = new CarbonFile[listStatus.length];
- for (int i = 0; i < files.length; i++) {
- files[i] = new HDFSCarbonFile(listStatus[i]);
- }
- return files;
- }
-
- @Override
- public CarbonFile[] listFiles() {
- FileStatus[] listStatus = null;
- try {
- if (null != fileStatus && fileStatus.isDirectory()) {
- Path path = fileStatus.getPath();
- listStatus = path.getFileSystem(FileFactory.getConfiguration()).listStatus(path);
- } else {
- return null;
- }
- } catch (IOException e) {
- LOGGER.error("Exception occured: " + e.getMessage());
- return new CarbonFile[0];
- }
- return getFiles(listStatus);
- }
-
- @Override
- public CarbonFile[] listFiles(final CarbonFileFilter fileFilter) {
- CarbonFile[] files = listFiles();
- if (files != null && files.length >= 1) {
- List<CarbonFile> fileList = new ArrayList<CarbonFile>(files.length);
- for (int i = 0; i < files.length; i++) {
- if (fileFilter.accept(files[i])) {
- fileList.add(files[i]);
- }
- }
- if (fileList.size() >= 1) {
- return fileList.toArray(new CarbonFile[fileList.size()]);
- } else {
- return new CarbonFile[0];
- }
- }
- return files;
- }
-
- @Override
- public CarbonFile getParentFile() {
- Path parent = fileStatus.getPath().getParent();
- return null == parent ? null : new HDFSCarbonFile(parent);
- }
-
- @Override
- public boolean renameForce(String changetoName) {
- FileSystem fs;
- try {
- fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
- if (fs instanceof DistributedFileSystem) {
- ((DistributedFileSystem) fs).rename(fileStatus.getPath(), new Path(changetoName),
- org.apache.hadoop.fs.Options.Rename.OVERWRITE);
- return true;
- } else {
- return false;
- }
- } catch (IOException e) {
- LOGGER.error("Exception occured: " + e.getMessage());
- return false;
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/LocalCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/LocalCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/LocalCarbonFile.java
deleted file mode 100644
index d7866c1..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/LocalCarbonFile.java
+++ /dev/null
@@ -1,229 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.filesystem;
-
-import java.io.File;
-import java.io.FileFilter;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.channels.FileChannel;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
-import org.apache.carbondata.core.util.CarbonUtil;
-
-import org.apache.hadoop.fs.Path;
-
-public class LocalCarbonFile implements CarbonFile {
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(LocalCarbonFile.class.getName());
- private File file;
-
- public LocalCarbonFile(String filePath) {
- Path pathWithoutSchemeAndAuthority = Path.getPathWithoutSchemeAndAuthority(new Path(filePath));
- file = new File(pathWithoutSchemeAndAuthority.toString());
- }
-
- public LocalCarbonFile(File file) {
- this.file = file;
- }
-
- @Override public String getAbsolutePath() {
- return file.getAbsolutePath();
- }
-
- @Override public CarbonFile[] listFiles(final CarbonFileFilter fileFilter) {
- if (!file.isDirectory()) {
- return null;
- }
-
- File[] files = file.listFiles(new FileFilter() {
-
- @Override public boolean accept(File pathname) {
- return fileFilter.accept(new LocalCarbonFile(pathname));
- }
- });
-
- if (files == null) {
- return new CarbonFile[0];
- }
-
- CarbonFile[] carbonFiles = new CarbonFile[files.length];
-
- for (int i = 0; i < carbonFiles.length; i++) {
- carbonFiles[i] = new LocalCarbonFile(files[i]);
- }
-
- return carbonFiles;
- }
-
- @Override public String getName() {
- return file.getName();
- }
-
- @Override public boolean isDirectory() {
- return file.isDirectory();
- }
-
- @Override public boolean exists() {
- if (file != null) {
- return file.exists();
- }
- return false;
- }
-
- @Override public String getCanonicalPath() {
- try {
- return file.getCanonicalPath();
- } catch (IOException e) {
- LOGGER
- .error(e, "Exception occured" + e.getMessage());
- }
- return null;
- }
-
- @Override public CarbonFile getParentFile() {
- return new LocalCarbonFile(file.getParentFile());
- }
-
- @Override public String getPath() {
- return file.getPath();
- }
-
- @Override public long getSize() {
- return file.length();
- }
-
- public boolean renameTo(String changetoName) {
- return file.renameTo(new File(changetoName));
- }
-
- public boolean delete() {
- return file.delete();
- }
-
- @Override public CarbonFile[] listFiles() {
-
- if (!file.isDirectory()) {
- return null;
- }
- File[] files = file.listFiles();
- if (files == null) {
- return new CarbonFile[0];
- }
- CarbonFile[] carbonFiles = new CarbonFile[files.length];
- for (int i = 0; i < carbonFiles.length; i++) {
- carbonFiles[i] = new LocalCarbonFile(files[i]);
- }
-
- return carbonFiles;
-
- }
-
- @Override public boolean createNewFile() {
- try {
- return file.createNewFile();
- } catch (IOException e) {
- return false;
- }
- }
-
- @Override public long getLastModifiedTime() {
- return file.lastModified();
- }
-
- @Override public boolean setLastModifiedTime(long timestamp) {
- return file.setLastModified(timestamp);
- }
-
- /**
- * This method will delete the data in file data from a given offset
- */
- @Override public boolean truncate(String fileName, long validDataEndOffset) {
- FileChannel source = null;
- FileChannel destination = null;
- boolean fileTruncatedSuccessfully = false;
- // temporary file name
- String tempWriteFilePath = fileName + CarbonCommonConstants.TEMPWRITEFILEEXTENSION;
- FileFactory.FileType fileType = FileFactory.getFileType(fileName);
- try {
- CarbonFile tempFile = null;
- // delete temporary file if it already exists at a given path
- if (FileFactory.isFileExist(tempWriteFilePath, fileType)) {
- tempFile = FileFactory.getCarbonFile(tempWriteFilePath, fileType);
- tempFile.delete();
- }
- // create new temporary file
- FileFactory.createNewFile(tempWriteFilePath, fileType);
- tempFile = FileFactory.getCarbonFile(tempWriteFilePath, fileType);
- source = new FileInputStream(fileName).getChannel();
- destination = new FileOutputStream(tempWriteFilePath).getChannel();
- long read = destination.transferFrom(source, 0, validDataEndOffset);
- long totalBytesRead = read;
- long remaining = validDataEndOffset - totalBytesRead;
- // read till required data offset is not reached
- while (remaining > 0) {
- read = destination.transferFrom(source, totalBytesRead, remaining);
- totalBytesRead = totalBytesRead + read;
- remaining = remaining - totalBytesRead;
- }
- CarbonUtil.closeStreams(source, destination);
- // rename the temp file to original file
- tempFile.renameForce(fileName);
- fileTruncatedSuccessfully = true;
- } catch (IOException e) {
- LOGGER.error("Exception occured while truncating the file " + e.getMessage());
- } finally {
- CarbonUtil.closeStreams(source, destination);
- }
- return fileTruncatedSuccessfully;
- }
-
- /**
- * This method will be used to check whether a file has been modified or not
- *
- * @param fileTimeStamp time to be compared with latest timestamp of file
- * @param endOffset file length to be compared with current length of file
- * @return
- */
- @Override public boolean isFileModified(long fileTimeStamp, long endOffset) {
- boolean isFileModified = false;
- if (getLastModifiedTime() > fileTimeStamp || getSize() > endOffset) {
- isFileModified = true;
- }
- return isFileModified;
- }
-
- @Override public boolean renameForce(String changetoName) {
- File destFile = new File(changetoName);
- if (destFile.exists()) {
- if (destFile.delete()) {
- return file.renameTo(new File(changetoName));
- }
- }
-
- return file.renameTo(new File(changetoName));
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/ViewFSCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/ViewFSCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/ViewFSCarbonFile.java
deleted file mode 100644
index 3fcf387..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/ViewFSCarbonFile.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.datastorage.filesystem;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.viewfs.ViewFileSystem;
-
-public class ViewFSCarbonFile extends AbstractDFSCarbonFile {
- /**
- * LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(ViewFSCarbonFile.class.getName());
-
- public ViewFSCarbonFile(String filePath) {
- super(filePath);
- }
-
- public ViewFSCarbonFile(Path path) {
- super(path);
- }
-
- public ViewFSCarbonFile(FileStatus fileStatus) {
- super(fileStatus);
- }
-
- /**
- * @param listStatus
- * @return
- */
- private CarbonFile[] getFiles(FileStatus[] listStatus) {
- if (listStatus == null) {
- return new CarbonFile[0];
- }
- CarbonFile[] files = new CarbonFile[listStatus.length];
- for (int i = 0; i < files.length; i++) {
- files[i] = new ViewFSCarbonFile(listStatus[i]);
- }
- return files;
- }
-
- @Override
- public CarbonFile[] listFiles() {
- FileStatus[] listStatus = null;
- try {
- if (null != fileStatus && fileStatus.isDirectory()) {
- Path path = fileStatus.getPath();
- listStatus = path.getFileSystem(FileFactory.getConfiguration()).listStatus(path);
- } else {
- return null;
- }
- } catch (IOException ex) {
- LOGGER.error("Exception occured" + ex.getMessage());
- return new CarbonFile[0];
- }
- return getFiles(listStatus);
- }
-
- @Override
- public CarbonFile[] listFiles(final CarbonFileFilter fileFilter) {
- CarbonFile[] files = listFiles();
- if (files != null && files.length >= 1) {
- List<CarbonFile> fileList = new ArrayList<CarbonFile>(files.length);
- for (int i = 0; i < files.length; i++) {
- if (fileFilter.accept(files[i])) {
- fileList.add(files[i]);
- }
- }
- if (fileList.size() >= 1) {
- return fileList.toArray(new CarbonFile[fileList.size()]);
- } else {
- return new CarbonFile[0];
- }
- }
- return files;
- }
-
- @Override public CarbonFile getParentFile() {
- Path parent = fileStatus.getPath().getParent();
- return null == parent ? null : new ViewFSCarbonFile(parent);
- }
-
- @Override
- public boolean renameForce(String changetoName) {
- FileSystem fs;
- try {
- fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
- if (fs instanceof ViewFileSystem) {
- fs.delete(new Path(changetoName), true);
- fs.rename(fileStatus.getPath(), new Path(changetoName));
- return true;
- } else {
- return false;
- }
- } catch (IOException e) {
- LOGGER.error("Exception occured" + e.getMessage());
- return false;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/impl/CompressedDataMeasureDataWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/CompressedDataMeasureDataWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/CompressedDataMeasureDataWrapper.java
deleted file mode 100644
index 997543b..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/CompressedDataMeasureDataWrapper.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.impl;
-
-import org.apache.carbondata.core.datastorage.MeasureDataWrapper;
-import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder;
-
-public class CompressedDataMeasureDataWrapper implements MeasureDataWrapper {
-
- private final CarbonReadDataHolder[] values;
-
- public CompressedDataMeasureDataWrapper(final CarbonReadDataHolder[] values) {
- this.values = values;
- }
-
- @Override public CarbonReadDataHolder[] getValues() {
- return values;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/impl/DFSFileHolderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/DFSFileHolderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/DFSFileHolderImpl.java
deleted file mode 100644
index 443344f..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/DFSFileHolderImpl.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.datastorage.impl;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.FileHolder;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-
-public class DFSFileHolderImpl implements FileHolder {
- /**
- * cache to hold filename and its stream
- */
- private Map<String, FSDataInputStream> fileNameAndStreamCache;
-
- public DFSFileHolderImpl() {
- this.fileNameAndStreamCache =
- new HashMap<String, FSDataInputStream>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- }
-
- @Override public byte[] readByteArray(String filePath, long offset, int length)
- throws IOException {
- FSDataInputStream fileChannel = updateCache(filePath);
- return read(fileChannel, length, offset);
- }
-
- /**
- * This method will be used to check whether stream is already present in
- * cache or not for filepath if not present then create it and then add to
- * cache, other wise get from cache
- *
- * @param filePath fully qualified file path
- * @return channel
- */
- private FSDataInputStream updateCache(String filePath) throws IOException {
- FSDataInputStream fileChannel = fileNameAndStreamCache.get(filePath);
- if (null == fileChannel) {
- Path pt = new Path(filePath);
- FileSystem fs = FileSystem.get(FileFactory.getConfiguration());
- fileChannel = fs.open(pt);
- fileNameAndStreamCache.put(filePath, fileChannel);
- }
- return fileChannel;
- }
-
- /**
- * This method will be used to read from file based on number of bytes to be read and positon
- *
- * @param channel file channel
- * @param size number of bytes
- * @param offset position
- * @return byte buffer
- */
- private byte[] read(FSDataInputStream channel, int size, long offset) throws IOException {
- byte[] byteBffer = new byte[size];
- channel.seek(offset);
- channel.readFully(byteBffer);
- return byteBffer;
- }
-
- /**
- * This method will be used to read from file based on number of bytes to be read and positon
- *
- * @param channel file channel
- * @param size number of bytes
- * @return byte buffer
- */
- private byte[] read(FSDataInputStream channel, int size) throws IOException {
- byte[] byteBffer = new byte[size];
- channel.readFully(byteBffer);
- return byteBffer;
- }
-
- @Override public int readInt(String filePath, long offset) throws IOException {
- FSDataInputStream fileChannel = updateCache(filePath);
- fileChannel.seek(offset);
- return fileChannel.readInt();
- }
-
- @Override public long readDouble(String filePath, long offset) throws IOException {
- FSDataInputStream fileChannel = updateCache(filePath);
- fileChannel.seek(offset);
- return fileChannel.readLong();
- }
-
- @Override public void finish() throws IOException {
- for (Entry<String, FSDataInputStream> entry : fileNameAndStreamCache.entrySet()) {
- FSDataInputStream channel = entry.getValue();
- if (null != channel) {
- channel.close();
- }
- }
- }
-
- @Override public byte[] readByteArray(String filePath, int length) throws IOException {
- FSDataInputStream fileChannel = updateCache(filePath);
- return read(fileChannel, length);
- }
-
- @Override public long readLong(String filePath, long offset) throws IOException {
- FSDataInputStream fileChannel = updateCache(filePath);
- fileChannel.seek(offset);
- return fileChannel.readLong();
- }
-
- @Override public int readInt(String filePath) throws IOException {
- FSDataInputStream fileChannel = updateCache(filePath);
- return fileChannel.readInt();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/impl/FileFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/FileFactory.java b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/FileFactory.java
deleted file mode 100644
index ca9956d..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/FileFactory.java
+++ /dev/null
@@ -1,485 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.impl;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.zip.GZIPInputStream;
-
-import org.apache.carbondata.core.datastorage.FileHolder;
-import org.apache.carbondata.core.datastorage.filesystem.AlluxioCarbonFile;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.filesystem.HDFSCarbonFile;
-import org.apache.carbondata.core.datastorage.filesystem.LocalCarbonFile;
-import org.apache.carbondata.core.datastorage.filesystem.ViewFSCarbonFile;
-import org.apache.carbondata.core.util.CarbonUtil;
-
-import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.compress.BZip2Codec;
-import org.apache.hadoop.io.compress.GzipCodec;
-
-public final class FileFactory {
- private static Configuration configuration = null;
-
- static {
- configuration = new Configuration();
- configuration.addResource(new Path("../core-default.xml"));
- }
-
- private FileFactory() {
-
- }
-
- public static Configuration getConfiguration() {
- return configuration;
- }
-
- public static FileHolder getFileHolder(FileType fileType) {
- switch (fileType) {
- case LOCAL:
- return new FileHolderImpl();
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- return new DFSFileHolderImpl();
- default:
- return new FileHolderImpl();
- }
- }
-
- public static FileType getFileType(String path) {
- if (path.startsWith(CarbonUtil.HDFS_PREFIX)) {
- return FileType.HDFS;
- }
- else if (path.startsWith(CarbonUtil.ALLUXIO_PREFIX)) {
- return FileType.ALLUXIO;
- }
- else if (path.startsWith(CarbonUtil.VIEWFS_PREFIX)) {
- return FileType.VIEWFS;
- }
- return FileType.LOCAL;
- }
-
- public static CarbonFile getCarbonFile(String path, FileType fileType) {
- switch (fileType) {
- case LOCAL:
- return new LocalCarbonFile(path);
- case HDFS:
- return new HDFSCarbonFile(path);
- case ALLUXIO:
- return new AlluxioCarbonFile(path);
- case VIEWFS:
- return new ViewFSCarbonFile(path);
- default:
- return new LocalCarbonFile(path);
- }
- }
-
- public static DataInputStream getDataInputStream(String path, FileType fileType)
- throws IOException {
- return getDataInputStream(path, fileType, -1);
- }
-
- public static DataInputStream getDataInputStream(String path, FileType fileType, int bufferSize)
- throws IOException {
- path = path.replace("\\", "/");
- boolean gzip = path.endsWith(".gz");
- boolean bzip2 = path.endsWith(".bz2");
- InputStream stream;
- switch (fileType) {
- case LOCAL:
- if (gzip) {
- stream = new GZIPInputStream(new FileInputStream(path));
- } else if (bzip2) {
- stream = new BZip2CompressorInputStream(new FileInputStream(path));
- } else {
- stream = new FileInputStream(path);
- }
- break;
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path pt = new Path(path);
- FileSystem fs = pt.getFileSystem(configuration);
- if (bufferSize == -1) {
- stream = fs.open(pt);
- } else {
- stream = fs.open(pt, bufferSize);
- }
- if (gzip) {
- GzipCodec codec = new GzipCodec();
- stream = codec.createInputStream(stream);
- } else if (bzip2) {
- BZip2Codec codec = new BZip2Codec();
- stream = codec.createInputStream(stream);
- }
- break;
- default:
- throw new UnsupportedOperationException("unsupported file system");
- }
- return new DataInputStream(new BufferedInputStream(stream));
- }
-
- /**
- * return the datainputStream which is seek to the offset of file
- *
- * @param path
- * @param fileType
- * @param bufferSize
- * @param offset
- * @return DataInputStream
- * @throws IOException
- */
- public static DataInputStream getDataInputStream(String path, FileType fileType, int bufferSize,
- long offset) throws IOException {
- path = path.replace("\\", "/");
- switch (fileType) {
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path pt = new Path(path);
- FileSystem fs = pt.getFileSystem(configuration);
- FSDataInputStream stream = fs.open(pt, bufferSize);
- stream.seek(offset);
- return new DataInputStream(new BufferedInputStream(stream));
- default:
- FileInputStream fis = new FileInputStream(path);
- long actualSkipSize = 0;
- long skipSize = offset;
- while (actualSkipSize != offset) {
- actualSkipSize += fis.skip(skipSize);
- skipSize = skipSize - actualSkipSize;
- }
- return new DataInputStream(new BufferedInputStream(fis));
- }
- }
-
- public static DataOutputStream getDataOutputStream(String path, FileType fileType)
- throws IOException {
- path = path.replace("\\", "/");
- switch (fileType) {
- case LOCAL:
- return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path)));
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path pt = new Path(path);
- FileSystem fs = pt.getFileSystem(configuration);
- FSDataOutputStream stream = fs.create(pt, true);
- return stream;
- default:
- return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path)));
- }
- }
-
- public static DataOutputStream getDataOutputStream(String path, FileType fileType, int bufferSize,
- boolean append) throws IOException {
- path = path.replace("\\", "/");
- switch (fileType) {
- case LOCAL:
- return new DataOutputStream(
- new BufferedOutputStream(new FileOutputStream(path, append), bufferSize));
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path pt = new Path(path);
- FileSystem fs = pt.getFileSystem(configuration);
- FSDataOutputStream stream = null;
- if (append) {
- // append to a file only if file already exists else file not found
- // exception will be thrown by hdfs
- if (CarbonUtil.isFileExists(path)) {
- stream = fs.append(pt, bufferSize);
- } else {
- stream = fs.create(pt, true, bufferSize);
- }
- } else {
- stream = fs.create(pt, true, bufferSize);
- }
- return stream;
- default:
- return new DataOutputStream(
- new BufferedOutputStream(new FileOutputStream(path), bufferSize));
- }
- }
-
- public static DataOutputStream getDataOutputStream(String path, FileType fileType, int bufferSize,
- long blockSize) throws IOException {
- path = path.replace("\\", "/");
- switch (fileType) {
- case LOCAL:
- return new DataOutputStream(
- new BufferedOutputStream(new FileOutputStream(path), bufferSize));
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path pt = new Path(path);
- FileSystem fs = pt.getFileSystem(configuration);
- FSDataOutputStream stream =
- fs.create(pt, true, bufferSize, fs.getDefaultReplication(pt), blockSize);
- return stream;
- default:
- return new DataOutputStream(
- new BufferedOutputStream(new FileOutputStream(path), bufferSize));
- }
- }
-
- /**
- * This method checks the given path exists or not and also is it file or
- * not if the performFileCheck is true
- *
- * @param filePath - Path
- * @param fileType - FileType Local/HDFS
- * @param performFileCheck - Provide false for folders, true for files and
- */
- public static boolean isFileExist(String filePath, FileType fileType, boolean performFileCheck)
- throws IOException {
- filePath = filePath.replace("\\", "/");
- switch (fileType) {
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path path = new Path(filePath);
- FileSystem fs = path.getFileSystem(configuration);
- if (performFileCheck) {
- return fs.exists(path) && fs.isFile(path);
- } else {
- return fs.exists(path);
- }
-
- case LOCAL:
- default:
- File defaultFile = new File(filePath);
-
- if (performFileCheck) {
- return defaultFile.exists() && defaultFile.isFile();
- } else {
- return defaultFile.exists();
- }
- }
- }
-
- /**
- * This method checks the given path exists or not and also is it file or
- * not if the performFileCheck is true
- *
- * @param filePath - Path
- * @param fileType - FileType Local/HDFS
- */
- public static boolean isFileExist(String filePath, FileType fileType) throws IOException {
- filePath = filePath.replace("\\", "/");
- switch (fileType) {
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path path = new Path(filePath);
- FileSystem fs = path.getFileSystem(configuration);
- return fs.exists(path);
-
- case LOCAL:
- default:
- File defaultFile = new File(filePath);
- return defaultFile.exists();
- }
- }
-
- public static boolean createNewFile(String filePath, FileType fileType) throws IOException {
- filePath = filePath.replace("\\", "/");
- switch (fileType) {
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path path = new Path(filePath);
- FileSystem fs = path.getFileSystem(configuration);
- return fs.createNewFile(path);
-
- case LOCAL:
- default:
- File file = new File(filePath);
- return file.createNewFile();
- }
- }
-
- public static boolean deleteFile(String filePath, FileType fileType) throws IOException {
- filePath = filePath.replace("\\", "/");
- switch (fileType) {
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path path = new Path(filePath);
- FileSystem fs = path.getFileSystem(configuration);
- return fs.delete(path, true);
-
- case LOCAL:
- default:
- File file = new File(filePath);
- return deleteAllFilesOfDir(file);
- }
- }
-
- public static boolean deleteAllFilesOfDir(File path) {
- if (!path.exists()) {
- return true;
- }
- if (path.isFile()) {
- return path.delete();
- }
- File[] files = path.listFiles();
- for (int i = 0; i < files.length; i++) {
- deleteAllFilesOfDir(files[i]);
- }
- return path.delete();
- }
-
- public static boolean mkdirs(String filePath, FileType fileType) throws IOException {
- filePath = filePath.replace("\\", "/");
- switch (fileType) {
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path path = new Path(filePath);
- FileSystem fs = path.getFileSystem(configuration);
- return fs.mkdirs(path);
- case LOCAL:
- default:
- File file = new File(filePath);
- return file.mkdirs();
- }
- }
-
- /**
- * for getting the dataoutput stream using the hdfs filesystem append API.
- *
- * @param path
- * @param fileType
- * @return
- * @throws IOException
- */
- public static DataOutputStream getDataOutputStreamUsingAppend(String path, FileType fileType)
- throws IOException {
- path = path.replace("\\", "/");
- switch (fileType) {
- case LOCAL:
- return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path, true)));
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path pt = new Path(path);
- FileSystem fs = pt.getFileSystem(configuration);
- FSDataOutputStream stream = fs.append(pt);
- return stream;
- default:
- return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path)));
- }
- }
-
- /**
- * for creating a new Lock file and if it is successfully created
- * then in case of abrupt shutdown then the stream to that file will be closed.
- *
- * @param filePath
- * @param fileType
- * @return
- * @throws IOException
- */
- public static boolean createNewLockFile(String filePath, FileType fileType) throws IOException {
- filePath = filePath.replace("\\", "/");
- switch (fileType) {
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path path = new Path(filePath);
- FileSystem fs = path.getFileSystem(configuration);
- if (fs.createNewFile(path)) {
- fs.deleteOnExit(path);
- return true;
- }
- return false;
- case LOCAL:
- default:
- File file = new File(filePath);
- return file.createNewFile();
- }
- }
-
- public enum FileType {
- LOCAL, HDFS, ALLUXIO, VIEWFS
- }
-
- /**
- * below method will be used to update the file path
- * for local type
- * it removes the file:/ from the path
- *
- * @param filePath
- * @return updated file path without url for local
- */
- public static String getUpdatedFilePath(String filePath) {
- FileType fileType = getFileType(filePath);
- switch (fileType) {
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- return filePath;
- case LOCAL:
- default:
- Path pathWithoutSchemeAndAuthority =
- Path.getPathWithoutSchemeAndAuthority(new Path(filePath));
- return pathWithoutSchemeAndAuthority.toString();
- }
- }
-
- /**
- * It computes size of directory
- *
- * @param filePath
- * @return size in bytes
- * @throws IOException
- */
- public static long getDirectorySize(String filePath) throws IOException {
- FileType fileType = getFileType(filePath);
- switch (fileType) {
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path path = new Path(filePath);
- FileSystem fs = path.getFileSystem(configuration);
- return fs.getContentSummary(path).getLength();
- case LOCAL:
- default:
- File file = new File(filePath);
- return FileUtils.sizeOfDirectory(file);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/impl/FileHolderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/FileHolderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/FileHolderImpl.java
deleted file mode 100644
index 843ec5a..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/FileHolderImpl.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.impl;
-
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.FileHolder;
-
-public class FileHolderImpl implements FileHolder {
- /**
- * cache to hold filename and its stream
- */
- private Map<String, FileChannel> fileNameAndStreamCache;
-
- /**
- * FileHolderImpl Constructor
- * It will create the cache
- */
- public FileHolderImpl() {
- this.fileNameAndStreamCache =
- new HashMap<String, FileChannel>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- }
-
- public FileHolderImpl(int capacity) {
- this.fileNameAndStreamCache = new HashMap<String, FileChannel>(capacity);
- }
-
- /**
- * This method will be used to read the byte array from file based on offset
- * and length(number of bytes) need to read
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @param length number of bytes to be read
- * @return read byte array
- */
- @Override public byte[] readByteArray(String filePath, long offset, int length)
- throws IOException {
- FileChannel fileChannel = updateCache(filePath);
- ByteBuffer byteBffer = read(fileChannel, length, offset);
- return byteBffer.array();
- }
-
- /**
- * This method will be used to close all the streams currently present in the cache
- */
- @Override public void finish() throws IOException {
- for (Entry<String, FileChannel> entry : fileNameAndStreamCache.entrySet()) {
- FileChannel channel = entry.getValue();
- if (null != channel) {
- channel.close();
- }
- }
- }
-
- /**
- * This method will be used to read int from file from postion(offset), here
- * length will be always 4 bacause int byte size if 4
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @return read int
- */
- @Override public int readInt(String filePath, long offset) throws IOException {
- FileChannel fileChannel = updateCache(filePath);
- ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.INT_SIZE_IN_BYTE, offset);
- return byteBffer.getInt();
- }
-
- /**
- * This method will be used to read int from file from postion(offset), here
- * length will be always 4 bacause int byte size if 4
- *
- * @param filePath fully qualified file path
- * @return read int
- */
- @Override public int readInt(String filePath) throws IOException {
- FileChannel fileChannel = updateCache(filePath);
- ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.INT_SIZE_IN_BYTE);
- return byteBffer.getInt();
- }
-
- /**
- * This method will be used to read int from file from postion(offset), here
- * length will be always 4 bacause int byte size if 4
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @return read int
- */
- @Override public long readDouble(String filePath, long offset) throws IOException {
- FileChannel fileChannel = updateCache(filePath);
- ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.LONG_SIZE_IN_BYTE, offset);
- return byteBffer.getLong();
- }
-
- /**
- * This method will be used to check whether stream is already present in
- * cache or not for filepath if not present then create it and then add to
- * cache, other wise get from cache
- *
- * @param filePath fully qualified file path
- * @return channel
- */
- private FileChannel updateCache(String filePath) throws FileNotFoundException {
- FileChannel fileChannel = fileNameAndStreamCache.get(filePath);
- if (null == fileChannel) {
- FileInputStream stream = new FileInputStream(filePath);
- fileChannel = stream.getChannel();
- fileNameAndStreamCache.put(filePath, fileChannel);
- }
- return fileChannel;
- }
-
- /**
- * This method will be used to read from file based on number of bytes to be read and positon
- *
- * @param channel file channel
- * @param size number of bytes
- * @param offset position
- * @return byte buffer
- */
- private ByteBuffer read(FileChannel channel, int size, long offset) throws IOException {
- ByteBuffer byteBffer = ByteBuffer.allocate(size);
- channel.position(offset);
- channel.read(byteBffer);
- byteBffer.rewind();
- return byteBffer;
- }
-
- /**
- * This method will be used to read from file based on number of bytes to be read and positon
- *
- * @param channel file channel
- * @param size number of bytes
- * @return byte buffer
- */
- private ByteBuffer read(FileChannel channel, int size) throws IOException {
- ByteBuffer byteBffer = ByteBuffer.allocate(size);
- channel.read(byteBffer);
- byteBffer.rewind();
- return byteBffer;
- }
-
-
- /**
- * This method will be used to read the byte array from file based on length(number of bytes)
- *
- * @param filePath fully qualified file path
- * @param length number of bytes to be read
- * @return read byte array
- */
- @Override public byte[] readByteArray(String filePath, int length) throws IOException {
- FileChannel fileChannel = updateCache(filePath);
- ByteBuffer byteBffer = read(fileChannel, length);
- return byteBffer.array();
- }
-
- /**
- * This method will be used to read long from file from postion(offset), here
- * length will be always 8 bacause int byte size is 8
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @return read long
- */
- @Override public long readLong(String filePath, long offset) throws IOException {
- FileChannel fileChannel = updateCache(filePath);
- ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.LONG_SIZE_IN_BYTE, offset);
- return byteBffer.getLong();
- }
-
-}
[02/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/RowResultMerger.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/RowResultMerger.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/RowResultMerger.java
index 2bbc72d..825c98d 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/RowResultMerger.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/RowResultMerger.java
@@ -46,8 +46,8 @@ import org.apache.carbondata.processing.store.CarbonFactDataHandlerColumnar;
import org.apache.carbondata.processing.store.CarbonFactDataHandlerModel;
import org.apache.carbondata.processing.store.CarbonFactHandler;
import org.apache.carbondata.processing.store.writer.exception.CarbonDataWriterException;
-import org.apache.carbondata.scan.result.iterator.RawResultIterator;
-import org.apache.carbondata.scan.wrappers.ByteArrayWrapper;
+import org.apache.carbondata.core.scan.result.iterator.RawResultIterator;
+import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
/**
* This is the Merger class responsible for the merging of the segments.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/TupleConversionAdapter.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/TupleConversionAdapter.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/TupleConversionAdapter.java
index 2a48bab..497605c 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/TupleConversionAdapter.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/TupleConversionAdapter.java
@@ -23,7 +23,7 @@ import java.util.List;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.processing.util.RemoveDictionaryUtil;
-import org.apache.carbondata.scan.wrappers.ByteArrayWrapper;
+import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
/**
* This class will be used to convert the Result into the format used in data writer.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark-common/src/main/java/org/apache/carbondata/spark/partition/api/impl/QueryPartitionHelper.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/partition/api/impl/QueryPartitionHelper.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/partition/api/impl/QueryPartitionHelper.java
index a864ebf..dd8556c 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/partition/api/impl/QueryPartitionHelper.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/partition/api/impl/QueryPartitionHelper.java
@@ -24,7 +24,7 @@ import java.util.List;
import java.util.Map;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.scan.model.CarbonQueryPlan;
+import org.apache.carbondata.core.scan.model.CarbonQueryPlan;
import org.apache.carbondata.spark.partition.api.DataPartitioner;
import org.apache.carbondata.spark.partition.api.Partition;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/CarbonQueryUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/CarbonQueryUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/CarbonQueryUtil.java
index 94e475e..ffce6cb 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/CarbonQueryUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/CarbonQueryUtil.java
@@ -27,7 +27,7 @@ import java.util.Map;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.load.LoadMetadataDetails;
-import org.apache.carbondata.scan.model.CarbonQueryPlan;
+import org.apache.carbondata.core.scan.model.CarbonQueryPlan;
import org.apache.carbondata.spark.partition.api.Partition;
import org.apache.carbondata.spark.partition.api.impl.DefaultLoadBalancer;
import org.apache.carbondata.spark.partition.api.impl.PartitionMultiFileImpl;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala
index 5104ff9..7f9320d 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala
@@ -27,9 +27,9 @@ import org.apache.spark.sql.types.StructType
import org.apache.carbondata.core.metadata.DataType
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn
-import org.apache.carbondata.scan.expression.{ColumnExpression => CarbonColumnExpression, Expression => CarbonExpression, LiteralExpression => CarbonLiteralExpression}
-import org.apache.carbondata.scan.expression.conditional._
-import org.apache.carbondata.scan.expression.logical.{AndExpression, FalseExpression, OrExpression}
+import org.apache.carbondata.core.scan.expression.{ColumnExpression => CarbonColumnExpression, Expression => CarbonExpression, LiteralExpression => CarbonLiteralExpression}
+import org.apache.carbondata.core.scan.expression.conditional._
+import org.apache.carbondata.core.scan.expression.logical.{AndExpression, FalseExpression, OrExpression}
import org.apache.carbondata.spark.util.CarbonScalaUtil
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark-common/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
index ce66293..bd8e54c 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
@@ -19,11 +19,11 @@ package org.apache.carbondata.spark
import scala.collection.mutable.HashMap
-import org.apache.carbondata.core.{CarbonTableIdentifier, ColumnIdentifier}
+import org.apache.carbondata.core.datastorage.filesystem.{CarbonFile, CarbonFileFilter}
+import org.apache.carbondata.core.datastorage.impl.FileFactory
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.path.{CarbonStorePath, CarbonTablePath}
-import org.apache.carbondata.core.datastorage.store.filesystem.{CarbonFile, CarbonFileFilter}
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory
+import org.apache.carbondata.core.{CarbonTableIdentifier, ColumnIdentifier}
class DictionaryDetailHelper extends DictionaryDetailService {
def getDictionaryDetail(dictfolderPath: String, primDimensions: Array[CarbonDimension],
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
index 1c4c47c..045ed2f 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
@@ -32,15 +32,14 @@ import org.apache.spark._
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.Row
-import org.apache.carbondata.common.factory.CarbonCommonFactory
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.{CarbonTableIdentifier, ColumnIdentifier}
import org.apache.carbondata.core.cache.dictionary.Dictionary
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.path.CarbonTablePath
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory
-import org.apache.carbondata.core.service.PathService
+import org.apache.carbondata.core.datastorage.impl.FileFactory
+import org.apache.carbondata.core.service.{CarbonCommonFactory, PathService}
import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory, CarbonUtil}
import org.apache.carbondata.locks.{CarbonLockFactory, LockUsage}
import org.apache.carbondata.processing.model.CarbonLoadModel
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index a4b9112..4acef95 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -39,14 +39,14 @@ import org.apache.carbondata.core.datastore.block.{Distributable, SegmentPropert
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter
import org.apache.carbondata.core.path.CarbonTablePath
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.update.UpdateVO
import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
import org.apache.carbondata.hadoop.{CarbonInputFormat, CarbonInputSplit, CarbonMultiBlockSplit}
import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil
import org.apache.carbondata.processing.model.CarbonLoadModel
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil
-import org.apache.carbondata.scan.result.iterator.RawResultIterator
+import org.apache.carbondata.core.scan.result.iterator.RawResultIterator
+import org.apache.carbondata.core.update.data.UpdateVO
import org.apache.carbondata.spark.MergeResult
import org.apache.carbondata.spark.load.CarbonLoaderUtil
import org.apache.carbondata.spark.merger.{CarbonCompactionExecutor, CarbonCompactionUtil, CarbonDataMergerUtil, CompactionType, RowResultMerger}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
index e93955e..2ac8ee3 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
@@ -37,8 +37,8 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.stats.{QueryStatistic, QueryStatisticsConstants, QueryStatisticsRecorder}
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory
import org.apache.carbondata.hadoop._
-import org.apache.carbondata.scan.expression.Expression
-import org.apache.carbondata.scan.model.QueryModel
+import org.apache.carbondata.core.scan.expression.Expression
+import org.apache.carbondata.core.scan.model.QueryModel
import org.apache.carbondata.spark.load.CarbonLoaderUtil
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark-common/src/main/scala/org/apache/carbondata/spark/tasks/DictionaryWriterTask.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/tasks/DictionaryWriterTask.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/tasks/DictionaryWriterTask.scala
index e23b58d..3b8f899 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/tasks/DictionaryWriterTask.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/tasks/DictionaryWriterTask.scala
@@ -20,9 +20,9 @@ import java.io.IOException
import scala.collection.mutable
-import org.apache.carbondata.common.factory.CarbonCommonFactory
import org.apache.carbondata.core.cache.dictionary.Dictionary
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.service.CarbonCommonFactory
import org.apache.carbondata.core.util.DataTypeUtil
import org.apache.carbondata.core.writer.CarbonDictionaryWriter
import org.apache.carbondata.spark.rdd.DictionaryLoadModel
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark-common/src/main/scala/org/apache/carbondata/spark/tasks/SortIndexWriterTask.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/tasks/SortIndexWriterTask.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/tasks/SortIndexWriterTask.scala
index d552331..1504b7a 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/tasks/SortIndexWriterTask.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/tasks/SortIndexWriterTask.scala
@@ -16,8 +16,8 @@
*/
package org.apache.carbondata.spark.tasks
-import org.apache.carbondata.common.factory.CarbonCommonFactory
import org.apache.carbondata.core.cache.dictionary.Dictionary
+import org.apache.carbondata.core.service.CarbonCommonFactory
import org.apache.carbondata.core.writer.sortindex.{CarbonDictionarySortIndexWriter, CarbonDictionarySortInfo, CarbonDictionarySortInfoPreparator}
import org.apache.carbondata.spark.rdd.DictionaryLoadModel
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
index ac0f757..6712f7e 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
@@ -26,7 +26,7 @@ import org.apache.spark.sql.types._
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.metadata.{DataType => CarbonDataType}
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory
+import org.apache.carbondata.core.datastorage.impl.FileFactory
import org.apache.carbondata.core.util.CarbonProperties
object CarbonScalaUtil {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
index cd943c8..013cb9b 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -36,7 +36,6 @@ import org.apache.spark.sql._
import org.apache.spark.sql.types.{StringType, StructField, StructType}
import org.apache.spark.util.FileUtils
-import org.apache.carbondata.common.factory.CarbonCommonFactory
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.CarbonTableIdentifier
import org.apache.carbondata.core.cache.dictionary.Dictionary
@@ -45,8 +44,9 @@ import org.apache.carbondata.core.metadata.Encoding
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.path.CarbonStorePath
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory
+import org.apache.carbondata.core.datastorage.impl.FileFactory
import org.apache.carbondata.core.reader.CarbonDictionaryReader
+import org.apache.carbondata.core.service.CarbonCommonFactory
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
import org.apache.carbondata.core.writer.CarbonDictionaryWriter
import org.apache.carbondata.hadoop.csv.CSVInputFormat
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 228ee95..406c94f 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -25,7 +25,6 @@ import scala.collection.mutable.Map
import org.apache.spark.sql.SQLContext
-import org.apache.carbondata.common.factory.CarbonCommonFactory
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.metadata.DataType
import org.apache.carbondata.core.metadata.Encoding
@@ -34,6 +33,7 @@ import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo,
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.load.LoadMetadataDetails
+import org.apache.carbondata.core.service.CarbonCommonFactory
import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager
import org.apache.carbondata.processing.model.CarbonLoadModel
import org.apache.carbondata.spark.CarbonSparkFactory
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala b/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala
index 387d1ef..7a19a92 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala
@@ -19,8 +19,8 @@ package org.apache.spark.util
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFile
+import org.apache.carbondata.core.datastorage.impl.FileFactory
import org.apache.carbondata.processing.etl.DataLoadingException
object FileUtils {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
index e532c6c..3bb1efd 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
@@ -22,7 +22,7 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, ExprId, LeafExpress
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.types.DataType
-import org.apache.carbondata.scan.expression.ColumnExpression
+import org.apache.carbondata.core.scan.expression.ColumnExpression
case class CarbonBoundReference(colExp: ColumnExpression, dataType: DataType, nullable: Boolean)
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
index 17f27cd..0ee44e1 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
@@ -39,7 +39,7 @@ import org.apache.carbondata.core.AbsoluteTableIdentifier
import org.apache.carbondata.core.path.CarbonTablePath
import org.apache.carbondata.hadoop.{CarbonInputFormat, CarbonInputSplit, CarbonProjection}
import org.apache.carbondata.hadoop.util.{CarbonInputFormatUtil, SchemaReader}
-import org.apache.carbondata.scan.expression.logical.AndExpression
+import org.apache.carbondata.core.scan.expression.logical.AndExpression
import org.apache.carbondata.spark.{CarbonFilters, CarbonOption}
import org.apache.carbondata.spark.merger.TableMeta
import org.apache.carbondata.spark.readsupport.SparkRowReadSupportImpl
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
index ff861e5..5bcaccd 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
@@ -29,10 +29,10 @@ import org.apache.spark.sql.hive.{CarbonMetaData, CarbonMetastoreTypes}
import org.apache.spark.sql.sources._
import org.apache.spark.sql.types.{DataType, StructType}
+import org.apache.carbondata.core.datastorage.impl.FileFactory
import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, CarbonDimension}
import org.apache.carbondata.core.metadata.schema.table.column.CarbonImplicitDimension
import org.apache.carbondata.core.path.CarbonStorePath
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory
import org.apache.carbondata.core.updatestatus.SegmentStatusManager
import org.apache.carbondata.spark.{CarbonOption, _}
import org.apache.carbondata.spark.merger.TableMeta
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
index 40a85a1..d080bde 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
@@ -27,7 +27,7 @@ import org.apache.spark.sql.execution.LeafNode
import org.apache.spark.sql.hive.CarbonMetastore
import org.apache.carbondata.hadoop.CarbonProjection
-import org.apache.carbondata.scan.model._
+import org.apache.carbondata.core.scan.model._
import org.apache.carbondata.spark.CarbonFilters
import org.apache.carbondata.spark.rdd.CarbonScanRDD
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala b/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
index 2c45524..d3bd4fb 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
@@ -22,10 +22,10 @@ import scala.collection.JavaConverters._
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.{Expression => SparkExpression, GenericMutableRow}
-import org.apache.carbondata.scan.expression.{ColumnExpression, ExpressionResult, UnknownExpression}
-import org.apache.carbondata.scan.expression.conditional.ConditionalExpression
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException
-import org.apache.carbondata.scan.filter.intf.{ExpressionType, RowIntf}
+import org.apache.carbondata.core.scan.expression.{ColumnExpression, ExpressionResult, UnknownExpression}
+import org.apache.carbondata.core.scan.expression.conditional.ConditionalExpression
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException
+import org.apache.carbondata.core.scan.filter.intf.{ExpressionType, RowIntf}
import org.apache.carbondata.spark.util.CarbonScalaUtil
class SparkUnknownExpression(var sparkExp: SparkExpression)
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
index f830782..b9bbe54 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
@@ -33,14 +33,14 @@ import org.apache.spark.sql.execution.datasources.LogicalRelation
import org.apache.spark.storage.StorageLevel
import org.apache.spark.util.FileUtils
-import org.apache.carbondata.common.iudprocessor.iuddata.RowCountDetailsVO
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.path.{CarbonStorePath, CarbonTablePath}
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory
-import org.apache.carbondata.core.update.{CarbonUpdateUtil, DeleteDeltaBlockDetails, SegmentUpdateDetails, TupleIdEnum}
+import org.apache.carbondata.core.datastorage.impl.FileFactory
+import org.apache.carbondata.core.update._
+import org.apache.carbondata.core.update.data.RowCountDetailsVO
import org.apache.carbondata.core.updatestatus.{SegmentStatusManager, SegmentUpdateStatusManager}
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.writer.CarbonDeleteDeltaWriterImpl
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index f9faaeb..b25731e 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -46,7 +46,7 @@ import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.path.CarbonStorePath
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory
+import org.apache.carbondata.core.datastorage.impl.FileFactory
import org.apache.carbondata.core.dictionary.server.DictionaryServer
import org.apache.carbondata.core.update.CarbonUpdateUtil
import org.apache.carbondata.core.update.TupleIdEnum
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
index f3099e8..4c6c5be 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
@@ -36,8 +36,8 @@ import org.apache.spark.sql.types._
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.CarbonTableIdentifier
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory.FileType
+import org.apache.carbondata.core.datastorage.impl.FileFactory
+import org.apache.carbondata.core.datastorage.impl.FileFactory.FileType
import org.apache.carbondata.core.metadata.CarbonMetadata
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSystemLockFeatureTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSystemLockFeatureTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSystemLockFeatureTest.scala
index bd95908..21a627e 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSystemLockFeatureTest.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSystemLockFeatureTest.scala
@@ -26,7 +26,7 @@ import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.core.path.{CarbonStorePath, CarbonTablePath}
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory
+import org.apache.carbondata.core.datastorage.impl.FileFactory
import org.apache.carbondata.core.updatestatus.SegmentStatusManager
import org.apache.carbondata.core.util.CarbonProperties
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
index 359e8f2..21c111d 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
@@ -27,10 +27,10 @@ import org.apache.spark.sql.common.util.QueryTest
import org.apache.spark.sql.{CarbonEnv, CarbonRelation}
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.common.ext.PathFactory
import org.apache.carbondata.core.CarbonDataLoadSchema
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory
+import org.apache.carbondata.core.datastorage.impl.FileFactory
+import org.apache.carbondata.core.service.impl.PathFactory
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.processing.constants.TableOptionConstant
import org.apache.carbondata.processing.model.CarbonLoadModel
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java b/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
index f5b1ad0..f4bb50a 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
@@ -20,8 +20,8 @@ package org.apache.carbondata.spark.readsupport;
import java.io.IOException;
-import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.hadoop.readsupport.impl.AbstractDictionaryDecodedReadSupport;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
index 5ed7389..76a827d 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/ColumnarVectorWrapper.java
@@ -16,7 +16,7 @@
*/
package org.apache.carbondata.spark.vectorreader;
-import org.apache.carbondata.scan.result.vector.CarbonColumnVector;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.spark.sql.execution.vectorized.ColumnVector;
import org.apache.spark.sql.types.Decimal;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
index 2b20d35..346ded0 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
@@ -24,22 +24,22 @@ import java.util.Map;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
-import org.apache.carbondata.core.metadata.datatype.DataType;
-import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.hadoop.CarbonInputSplit;
import org.apache.carbondata.hadoop.CarbonMultiBlockSplit;
-import org.apache.carbondata.scan.executor.QueryExecutor;
-import org.apache.carbondata.scan.executor.QueryExecutorFactory;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
-import org.apache.carbondata.scan.model.QueryDimension;
-import org.apache.carbondata.scan.model.QueryMeasure;
-import org.apache.carbondata.scan.model.QueryModel;
-import org.apache.carbondata.scan.result.iterator.AbstractDetailQueryResultIterator;
-import org.apache.carbondata.scan.result.vector.CarbonColumnVector;
-import org.apache.carbondata.scan.result.vector.CarbonColumnarBatch;
+import org.apache.carbondata.core.scan.executor.QueryExecutor;
+import org.apache.carbondata.core.scan.executor.QueryExecutorFactory;
+import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
+import org.apache.carbondata.core.scan.model.QueryDimension;
+import org.apache.carbondata.core.scan.model.QueryMeasure;
+import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.core.scan.result.iterator.AbstractDetailQueryResultIterator;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
import org.apache.carbondata.spark.util.CarbonScalaUtil;
import org.apache.hadoop.mapreduce.InputSplit;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala
index 419b845..c969b38 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala
@@ -24,12 +24,12 @@ import org.apache.spark.sql.optimizer.AttributeReferenceWrapper
import org.apache.spark.sql.sources
import org.apache.spark.sql.types.StructType
-import org.apache.carbondata.core.metadata.datatype.DataType
+import org.apache.carbondata.core.metadata.DataType
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn
-import org.apache.carbondata.scan.expression.{ColumnExpression => CarbonColumnExpression, Expression => CarbonExpression, LiteralExpression => CarbonLiteralExpression}
-import org.apache.carbondata.scan.expression.conditional._
-import org.apache.carbondata.scan.expression.logical.{AndExpression, FalseExpression, OrExpression}
+import org.apache.carbondata.core.scan.expression.{ColumnExpression => CarbonColumnExpression, Expression => CarbonExpression, LiteralExpression => CarbonLiteralExpression}
+import org.apache.carbondata.core.scan.expression.conditional._
+import org.apache.carbondata.core.scan.expression.logical.{AndExpression, FalseExpression, OrExpression}
import org.apache.carbondata.spark.util.CarbonScalaUtil
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonSparkFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonSparkFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonSparkFactory.scala
index c866311..4b409ed 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonSparkFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonSparkFactory.scala
@@ -16,7 +16,7 @@
*/
package org.apache.carbondata.spark
-import org.apache.carbondata.core.carbon.{CarbonTableIdentifier, ColumnIdentifier}
+import org.apache.carbondata.core.{CarbonTableIdentifier, ColumnIdentifier}
import org.apache.carbondata.core.metadata.schema.table.column.{CarbonDimension, ColumnSchema}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark2/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
index f37c02a..979e154 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
@@ -18,11 +18,11 @@ package org.apache.carbondata.spark
import scala.collection.mutable.HashMap
-import org.apache.carbondata.core.carbon.{CarbonTableIdentifier, ColumnIdentifier}
+import org.apache.carbondata.core.datastorage.filesystem.{CarbonFile, CarbonFileFilter}
+import org.apache.carbondata.core.datastorage.impl.FileFactory
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.path.{CarbonStorePath, CarbonTablePath}
-import org.apache.carbondata.core.datastorage.store.filesystem.{CarbonFile, CarbonFileFilter}
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory
+import org.apache.carbondata.core.{CarbonTableIdentifier, ColumnIdentifier}
class DictionaryDetailHelper extends DictionaryDetailService {
def getDictionaryDetail(dictfolderPath: String, primDimensions: Array[CarbonDimension],
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 6c587bb..f687589 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -38,7 +38,7 @@ import org.apache.spark.sql.hive.DistributionUtil
import org.apache.spark.util.SparkUtil
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.carbon.{CarbonDataLoadSchema, CarbonTableIdentifier, ColumnarFormatVersion}
+import org.apache.carbondata.core.{CarbonDataLoadSchema, CarbonTableIdentifier, ColumnarFormatVersion}
import org.apache.carbondata.core.datastore.block.{Distributable, TableBlockInfo}
import org.apache.carbondata.core.metadata.CarbonMetadata
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/CarbonSparkUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/CarbonSparkUtil.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/CarbonSparkUtil.scala
index 2f65fbc..fe8352a 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/CarbonSparkUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/CarbonSparkUtil.scala
@@ -21,7 +21,7 @@ import scala.collection.JavaConverters._
import org.apache.spark.sql.hive.{CarbonMetaData, DictionaryMap}
-import org.apache.carbondata.core.metadata.encoder.Encoding
+import org.apache.carbondata.core.metadata.Encoding
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.util.CarbonUtil
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala
index c55c807..b6fa827 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala
@@ -25,7 +25,7 @@ import org.apache.hadoop.mapred.JobConf
import org.apache.hadoop.mapreduce.Job
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat
-import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier
+import org.apache.carbondata.core.AbsoluteTableIdentifier
import org.apache.carbondata.hadoop.CarbonInputFormat
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
index 0d0e913..a733291 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
@@ -23,7 +23,7 @@ import org.apache.spark.sql.execution.command.LoadTable
import org.apache.spark.sql.types._
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.metadata.datatype.{DataType => CarbonType}
+import org.apache.carbondata.core.metadata.{DataType => CarbonType}
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.spark.CarbonOption
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
index 7087204..f5c99a5 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
@@ -27,12 +27,12 @@ import org.apache.spark.sql.hive.CarbonRelation
import org.apache.spark.sql.sources.{BaseRelation, Filter, InsertableRelation}
import org.apache.spark.sql.types.StructType
-import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier
+import org.apache.carbondata.core.AbsoluteTableIdentifier
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.hadoop.{CarbonInputFormat, CarbonProjection}
import org.apache.carbondata.hadoop.util.SchemaReader
-import org.apache.carbondata.scan.expression.Expression
-import org.apache.carbondata.scan.expression.logical.AndExpression
+import org.apache.carbondata.core.scan.expression.Expression
+import org.apache.carbondata.core.scan.expression.logical.AndExpression
import org.apache.carbondata.spark.CarbonFilters
import org.apache.carbondata.spark.merger.TableMeta
import org.apache.carbondata.spark.rdd.CarbonScanRDD
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
index d4c1cc4..f8c60a0 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
@@ -30,9 +30,9 @@ import org.apache.spark.sql.types._
import org.apache.carbondata.core.cache.{Cache, CacheProvider, CacheType}
import org.apache.carbondata.core.cache.dictionary.{Dictionary, DictionaryColumnUniqueIdentifier}
-import org.apache.carbondata.core.carbon.{AbsoluteTableIdentifier, ColumnIdentifier}
-import org.apache.carbondata.core.metadata.datatype.DataType
-import org.apache.carbondata.core.metadata.encoder.Encoding
+import org.apache.carbondata.core.{AbsoluteTableIdentifier, ColumnIdentifier}
+import org.apache.carbondata.core.metadata.DataType
+import org.apache.carbondata.core.metadata.Encoding
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.util.DataTypeUtil
import org.apache.carbondata.spark.CarbonAliasDecoderRelation
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonScan.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonScan.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonScan.scala
index 895718d..8f3dcdf 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonScan.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonScan.scala
@@ -25,7 +25,7 @@ import org.apache.spark.sql.hive.CarbonRelation
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.scan.model._
+import org.apache.carbondata.core.scan.model._
import org.apache.carbondata.spark.CarbonFilters
case class CarbonScan(
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark2/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
index 408da24..6adc701 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/SparkUnknownExpression.scala
@@ -24,11 +24,11 @@ import scala.collection.JavaConverters._
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.{Expression => SparkExpression, GenericInternalRow}
-import org.apache.carbondata.core.metadata.encoder.Encoding
-import org.apache.carbondata.scan.expression.{ColumnExpression, ExpressionResult, UnknownExpression}
-import org.apache.carbondata.scan.expression.conditional.ConditionalExpression
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException
-import org.apache.carbondata.scan.filter.intf.{ExpressionType, RowIntf}
+import org.apache.carbondata.core.metadata.Encoding
+import org.apache.carbondata.core.scan.expression.{ColumnExpression, ExpressionResult, UnknownExpression}
+import org.apache.carbondata.core.scan.expression.conditional.ConditionalExpression
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException
+import org.apache.carbondata.core.scan.filter.intf.{ExpressionType, RowIntf}
import org.apache.carbondata.spark.util.CarbonScalaUtil
class SparkUnknownExpression(var sparkExp: SparkExpression)
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
index f058550..dd12fd4 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
@@ -23,7 +23,7 @@ import scala.collection.mutable.{LinkedHashSet, Map}
import org.apache.spark.sql.execution.command.{BucketFields, ColumnProperty, Field, PartitionerField, TableModel}
-import org.apache.carbondata.core.metadata.datatype.DataType
+import org.apache.carbondata.core.metadata.DataType
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.DataTypeUtil
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 99bbe77..4894e47 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -37,14 +37,14 @@ import org.codehaus.jackson.map.ObjectMapper
import org.apache.carbondata.api.CarbonStore
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.carbon.{CarbonDataLoadSchema, CarbonTableIdentifier}
+import org.apache.carbondata.core.{CarbonDataLoadSchema, CarbonTableIdentifier}
import org.apache.carbondata.core.metadata.CarbonMetadata
-import org.apache.carbondata.core.metadata.encoder.Encoding
+import org.apache.carbondata.core.metadata.Encoding
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.path.CarbonStorePath
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory
+import org.apache.carbondata.core.datastorage.impl.FileFactory
import org.apache.carbondata.core.update.CarbonUpdateUtil
import org.apache.carbondata.core.update.TupleIdEnum
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
@@ -82,8 +82,7 @@ case class AlterTableCompaction(alterTableModel: AlterTableModel) extends Runnab
def run(sparkSession: SparkSession): Seq[Row] = {
val tableName = alterTableModel.tableName.toLowerCase
val databaseName = alterTableModel.dbName.getOrElse(sparkSession.catalog.currentDatabase)
- if (null == org.apache.carbondata.core.carbon.metadata.CarbonMetadata.getInstance
- .getCarbonTable(databaseName + "_" + tableName)) {
+ if (null == CarbonMetadata.getInstance.getCarbonTable(databaseName + "_" + tableName)) {
LOGGER.error(s"alter table failed. table not found: $databaseName.$tableName")
sys.error(s"alter table failed. table not found: $databaseName.$tableName")
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
index b10dacd..4faa8a9 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
@@ -18,36 +18,35 @@
package org.apache.spark.sql.hive
import java.io._
-import java.util.{GregorianCalendar, LinkedHashSet, UUID}
import java.util.concurrent.atomic.AtomicLong
+import java.util.{GregorianCalendar, LinkedHashSet, UUID}
import scala.Array.canBuildFrom
-import scala.collection.mutable.ArrayBuffer
import scala.collection.JavaConverters._
+import scala.collection.mutable.ArrayBuffer
import scala.util.parsing.combinator.RegexParsers
-import org.apache.spark.internal.Logging
-import org.apache.spark.sql.{RuntimeConfig, SparkSession}
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.analysis.{MultiInstanceRelation, NoSuchTableException}
import org.apache.spark.sql.catalyst.expressions.AttributeReference
import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, Statistics}
import org.apache.spark.sql.execution.command.Partitioner
import org.apache.spark.sql.types._
+import org.apache.spark.sql.{RuntimeConfig, SparkSession}
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.carbon.CarbonTableIdentifier
-import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.CarbonTableIdentifier
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFile
+import org.apache.carbondata.core.datastorage.impl.FileFactory
+import org.apache.carbondata.core.datastorage.impl.FileFactory.FileType
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
+import org.apache.carbondata.core.metadata.{CarbonMetadata, DataType => CarbonDataType}
import org.apache.carbondata.core.path.{CarbonStorePath, CarbonTablePath}
-import org.apache.carbondata.core.carbon.querystatistics.{QueryStatistic, QueryStatisticsConstants}
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory.FileType
import org.apache.carbondata.core.reader.ThriftReader
+import org.apache.carbondata.core.stats.{QueryStatistic, QueryStatisticsConstants}
import org.apache.carbondata.core.updatestatus.SegmentStatusManager
import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory, CarbonUtil}
import org.apache.carbondata.core.writer.ThriftWriter
@@ -273,9 +272,7 @@ class CarbonMetastore(conf: RuntimeConfig, val storePath: String) {
wrapperTableInfo
.setMetaDataFilepath(CarbonTablePath.getFolderContainingFile(schemaFilePath))
CarbonMetadata.getInstance().loadTableMetadata(wrapperTableInfo)
- val carbonTable =
- org.apache.carbondata.core.carbon.metadata.CarbonMetadata.getInstance()
- .getCarbonTable(tableUniqueName)
+ val carbonTable = CarbonMetadata.getInstance().getCarbonTable(tableUniqueName)
metaDataBuffer += new TableMeta(carbonTable.getCarbonTableIdentifier, storePath,
carbonTable)
}
@@ -429,10 +426,8 @@ class CarbonMetastore(conf: RuntimeConfig, val storePath: String) {
metadataToBeRemoved match {
case Some(tableMeta) =>
metadata.tablesMeta -= tableMeta
- org.apache.carbondata.core.carbon.metadata.CarbonMetadata.getInstance
- .removeTable(dbName + "_" + tableName)
- org.apache.carbondata.core.carbon.metadata.CarbonMetadata.getInstance
- .removeTable(dbName + "_" + tableName)
+ CarbonMetadata.getInstance.removeTable(dbName + "_" + tableName)
+ CarbonMetadata.getInstance.removeTable(dbName + "_" + tableName)
updateSchemasUpdatedTime(touchSchemaFileSystemTime(dbName, tableName))
case None =>
LOGGER.info(s"Metadata does not contain entry for table $tableName in database $dbName")
@@ -797,8 +792,7 @@ case class CarbonRelation(
def addDecimalScaleAndPrecision(dimval: CarbonDimension, dataType: String): String = {
var dType = dataType
- if (dimval.getDataType
- == org.apache.carbondata.core.carbon.metadata.datatype.DataType.DECIMAL) {
+ if (dimval.getDataType == CarbonDataType.DECIMAL) {
dType +=
"(" + dimval.getColumnSchema.getPrecision + "," + dimval.getColumnSchema.getScale + ")"
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala
index 52c0eb0..0488acd 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/optimizer/CarbonLateDecodeRule.scala
@@ -33,7 +33,7 @@ import org.apache.spark.sql.execution.datasources.LogicalRelation
import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.carbon.querystatistics.QueryStatistic
+import org.apache.carbondata.core.stats.QueryStatistic
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory
import org.apache.carbondata.spark.{CarbonAliasDecoderRelation, CarbonFilters}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java b/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
index 9ea3f65..1eddc32 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
@@ -29,10 +29,10 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.CarbonDataLoadSchema;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory.FileType;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.impl.FileFactory.FileType;
import org.apache.carbondata.processing.api.dataloader.SchemaInfo;
import org.apache.carbondata.processing.constants.DataProcessorConstants;
import org.apache.carbondata.processing.csvreaderstep.CsvInputMeta;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/processing/src/main/java/org/apache/carbondata/processing/csvload/GraphExecutionUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvload/GraphExecutionUtil.java b/processing/src/main/java/org/apache/carbondata/processing/csvload/GraphExecutionUtil.java
index eca7e19..fbaf622 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/csvload/GraphExecutionUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvload/GraphExecutionUtil.java
@@ -35,10 +35,10 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.CarbonDataLoadSchema;
import org.apache.carbondata.core.CarbonDataLoadSchema.DimensionRelation;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory.FileType;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.impl.FileFactory.FileType;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.processing.etl.DataLoadingException;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/UnivocityCsvParser.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/UnivocityCsvParser.java b/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/UnivocityCsvParser.java
index 2fa5219..486c416 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/UnivocityCsvParser.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/UnivocityCsvParser.java
@@ -27,8 +27,8 @@ import java.io.Reader;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory.FileType;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.impl.FileFactory.FileType;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
index 58aee78..e65846c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
@@ -30,8 +30,8 @@ import org.apache.carbondata.core.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.schema.BucketingInfo;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.partition.Partitioner;
-import org.apache.carbondata.core.partition.impl.HashPartitionerImpl;
+import org.apache.carbondata.processing.newflow.steps.partition.Partitioner;
+import org.apache.carbondata.processing.newflow.steps.partition.impl.HashPartitionerImpl;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.processing.constants.LoggerAction;
import org.apache.carbondata.processing.newflow.AbstractDataLoadProcessorStep;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/partition/Partitioner.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/partition/Partitioner.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/partition/Partitioner.java
new file mode 100644
index 0000000..95fb96b
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/partition/Partitioner.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.processing.newflow.steps.partition;
+
+/**
+ * Partitions the data as per key
+ */
+public interface Partitioner<Key> {
+
+ int getPartition(Key key);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/partition/impl/HashPartitionerImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/partition/impl/HashPartitionerImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/partition/impl/HashPartitionerImpl.java
new file mode 100644
index 0000000..4743d5c
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/partition/impl/HashPartitionerImpl.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.processing.newflow.steps.partition.impl;
+
+import java.util.List;
+
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.processing.newflow.steps.partition.Partitioner;
+
+/**
+ * Hash partitioner implementation
+ */
+public class HashPartitionerImpl implements Partitioner<Object[]> {
+
+ private int numberOfBuckets;
+
+ private Hash[] hashes;
+
+ public HashPartitionerImpl(List<Integer> indexes, List<ColumnSchema> columnSchemas,
+ int numberOfBuckets) {
+ this.numberOfBuckets = numberOfBuckets;
+ hashes = new Hash[indexes.size()];
+ for (int i = 0; i < indexes.size(); i++) {
+ switch(columnSchemas.get(i).getDataType()) {
+ case SHORT:
+ case INT:
+ case LONG:
+ hashes[i] = new IntegralHash(indexes.get(i));
+ break;
+ case DOUBLE:
+ case FLOAT:
+ case DECIMAL:
+ hashes[i] = new DecimalHash(indexes.get(i));
+ break;
+ default:
+ hashes[i] = new StringHash(indexes.get(i));
+ }
+ }
+ }
+
+ @Override public int getPartition(Object[] objects) {
+ int hashCode = 0;
+ for (Hash hash : hashes) {
+ hashCode += hash.getHash(objects);
+ }
+ return (hashCode & Integer.MAX_VALUE) % numberOfBuckets;
+ }
+
+ private interface Hash {
+ int getHash(Object[] value);
+ }
+
+ private static class IntegralHash implements Hash {
+
+ private int index;
+
+ private IntegralHash(int index) {
+ this.index = index;
+ }
+
+ public int getHash(Object[] value) {
+ return value[index] != null ? Long.valueOf(value[index].toString()).hashCode() : 0;
+ }
+ }
+
+ private static class DecimalHash implements Hash {
+
+ private int index;
+
+ private DecimalHash(int index) {
+ this.index = index;
+ }
+
+ public int getHash(Object[] value) {
+ return value[index] != null ? Double.valueOf(value[index].toString()).hashCode() : 0;
+ }
+ }
+
+ private static class StringHash implements Hash {
+
+ private int index;
+
+ private StringHash(int index) {
+ this.index = index;
+ }
+
+ @Override public int getHash(Object[] value) {
+ return value[index] != null ? value[index].hashCode() : 0;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/CompressedTempSortFileWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/CompressedTempSortFileWriter.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/CompressedTempSortFileWriter.java
index b2d3055..ab371fa 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/CompressedTempSortFileWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/CompressedTempSortFileWriter.java
@@ -24,7 +24,7 @@ import java.io.DataOutputStream;
import java.io.IOException;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
+import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
index 4d62635..1de837c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
@@ -47,13 +47,12 @@ import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.columnar.BlockIndexerStorageForInt;
-import org.apache.carbondata.core.datastorage.store.columnar.BlockIndexerStorageForNoInvertedIndex;
-import org.apache.carbondata.core.datastorage.store.columnar.ColumnGroupModel;
-import org.apache.carbondata.core.datastorage.store.columnar.IndexStorage;
-import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder;
-import org.apache.carbondata.core.datastorage.util.StoreFactory;
+import org.apache.carbondata.core.datastorage.columnar.BlockIndexerStorageForInt;
+import org.apache.carbondata.core.datastorage.columnar.BlockIndexerStorageForNoInvertedIndex;
+import org.apache.carbondata.core.datastorage.columnar.ColumnGroupModel;
+import org.apache.carbondata.core.datastorage.columnar.IndexStorage;
+import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.keygenerator.columnar.ColumnarSplitter;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/processing/src/main/java/org/apache/carbondata/processing/store/StoreFactory.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/StoreFactory.java b/processing/src/main/java/org/apache/carbondata/processing/store/StoreFactory.java
new file mode 100644
index 0000000..f1e4713
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/StoreFactory.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.processing.store;
+
+import org.apache.carbondata.core.datastorage.NodeMeasureDataStore;
+import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastorage.impl.data.compressed.HeavyCompressedDoubleArrayDataInMemoryStore;
+
+public final class StoreFactory {
+
+ private StoreFactory() {
+ }
+
+ public static NodeMeasureDataStore createDataStore(WriterCompressModel compressionModel) {
+ return new HeavyCompressedDoubleArrayDataInMemoryStore(compressionModel);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/processing/src/main/java/org/apache/carbondata/processing/store/colgroup/ColGroupBlockStorage.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/colgroup/ColGroupBlockStorage.java b/processing/src/main/java/org/apache/carbondata/processing/store/colgroup/ColGroupBlockStorage.java
index 64a3ee8..59383c4 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/colgroup/ColGroupBlockStorage.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/colgroup/ColGroupBlockStorage.java
@@ -20,7 +20,7 @@ package org.apache.carbondata.processing.store.colgroup;
import java.util.concurrent.Callable;
-import org.apache.carbondata.core.datastorage.store.columnar.IndexStorage;
+import org.apache.carbondata.core.datastorage.columnar.IndexStorage;
/**
* it is holder of column group data and also min max for colgroup block data
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index 7a60d29..2ef48af 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -52,10 +52,10 @@ import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.columnar.IndexStorage;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.columnar.IndexStorage;
+import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
import org.apache.carbondata.core.metadata.BlockletInfoColumnar;
import org.apache.carbondata.core.util.ByteUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java
index 6fa1522..4a45b28 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java
@@ -19,8 +19,8 @@
package org.apache.carbondata.processing.store.writer;
-import org.apache.carbondata.core.datastorage.store.columnar.IndexStorage;
-import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastorage.columnar.IndexStorage;
+import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
import org.apache.carbondata.processing.store.writer.exception.CarbonDataWriterException;
public interface CarbonFactDataWriter<T> {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/processing/src/main/java/org/apache/carbondata/processing/store/writer/NodeHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/NodeHolder.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/NodeHolder.java
index 30c9d7c..885ecb8 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/NodeHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/NodeHolder.java
@@ -21,7 +21,7 @@ package org.apache.carbondata.processing.store.writer;
import java.util.BitSet;
-import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
public class NodeHolder {
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
index 8d53fe4..24435e8 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
@@ -27,8 +27,8 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.columnar.IndexStorage;
-import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastorage.columnar.IndexStorage;
+import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
import org.apache.carbondata.core.metadata.BlockletInfoColumnar;
import org.apache.carbondata.core.util.CarbonMetadataUtil;
import org.apache.carbondata.core.writer.CarbonFooterWriter;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/BadRecordsLogger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/BadRecordsLogger.java b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/BadRecordsLogger.java
index bac2196..33430e7 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/BadRecordsLogger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/BadRecordsLogger.java
@@ -32,8 +32,8 @@ import java.util.Map;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory.FileType;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.impl.FileFactory.FileType;
import org.apache.carbondata.core.util.CarbonUtil;
public class BadRecordsLogger {
[44/57] [abbrv] incubator-carbondata git commit: fix style
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSystemLockFeatureTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSystemLockFeatureTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSystemLockFeatureTest.scala
index 52999db..3f266da 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSystemLockFeatureTest.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSystemLockFeatureTest.scala
@@ -25,7 +25,7 @@ import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.path.{CarbonStorePath, CarbonTablePath}
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastorage.impl.FileFactory
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.core.updatestatus.SegmentStatusManager
import org.apache.carbondata.core.util.CarbonProperties
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
index 295d700..70d5f5e 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
@@ -28,7 +28,7 @@ import org.apache.spark.sql.{CarbonEnv, CarbonRelation}
import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastorage.impl.FileFactory
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.service.impl.PathFactory
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.processing.constants.TableOptionConstant
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
index 346ded0..4dc97a2 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/spark/vectorreader/VectorizedCarbonRecordReader.java
@@ -24,13 +24,10 @@ import java.util.Map;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.hadoop.CarbonInputSplit;
-import org.apache.carbondata.hadoop.CarbonMultiBlockSplit;
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.scan.executor.QueryExecutor;
import org.apache.carbondata.core.scan.executor.QueryExecutorFactory;
import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
@@ -40,6 +37,9 @@ import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.core.scan.result.iterator.AbstractDetailQueryResultIterator;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.hadoop.CarbonInputSplit;
+import org.apache.carbondata.hadoop.CarbonMultiBlockSplit;
import org.apache.carbondata.spark.util.CarbonScalaUtil;
import org.apache.hadoop.mapreduce.InputSplit;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark2/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
index 087a62c..5eec901 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
@@ -18,8 +18,8 @@ package org.apache.carbondata.spark
import scala.collection.mutable.HashMap
-import org.apache.carbondata.core.datastorage.filesystem.{CarbonFile, CarbonFileFilter}
-import org.apache.carbondata.core.datastorage.impl.FileFactory
+import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.{CarbonTableIdentifier, ColumnIdentifier}
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.path.{CarbonStorePath, CarbonTablePath}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index e662f96..9e9ef9e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -36,7 +36,7 @@ import org.codehaus.jackson.map.ObjectMapper
import org.apache.carbondata.api.CarbonStore
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastorage.impl.FileFactory
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.{CarbonMetadata, CarbonTableIdentifier, Encoding}
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
index f299de9..5b77249 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
@@ -36,9 +36,9 @@ import org.apache.spark.sql.{RuntimeConfig, SparkSession}
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFile
-import org.apache.carbondata.core.datastorage.impl.FileFactory
-import org.apache.carbondata.core.datastorage.impl.FileFactory.FileType
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.datastore.impl.FileFactory.FileType
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java b/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
index 3599256..4e2251e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
@@ -27,17 +27,17 @@ import java.util.Set;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.processing.model.CarbonDataLoadSchema;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
-import org.apache.carbondata.core.datastorage.impl.FileFactory.FileType;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory.FileType;
import org.apache.carbondata.processing.api.dataloader.SchemaInfo;
import org.apache.carbondata.processing.constants.DataProcessorConstants;
import org.apache.carbondata.processing.csvreaderstep.CsvInputMeta;
import org.apache.carbondata.processing.dataprocessor.IDataProcessStatus;
import org.apache.carbondata.processing.etl.DataLoadingException;
+import org.apache.carbondata.processing.model.CarbonDataLoadSchema;
import org.apache.carbondata.processing.surrogatekeysgenerator.csvbased.BadRecordsLogger;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/csvload/GraphExecutionUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvload/GraphExecutionUtil.java b/processing/src/main/java/org/apache/carbondata/processing/csvload/GraphExecutionUtil.java
index d136575..ccff5b6 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/csvload/GraphExecutionUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvload/GraphExecutionUtil.java
@@ -32,15 +32,15 @@ import java.util.Set;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.processing.model.CarbonDataLoadSchema;
-import org.apache.carbondata.processing.model.CarbonDataLoadSchema.DimensionRelation;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
-import org.apache.carbondata.core.datastorage.impl.FileFactory.FileType;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory.FileType;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.processing.etl.DataLoadingException;
+import org.apache.carbondata.processing.model.CarbonDataLoadSchema;
+import org.apache.carbondata.processing.model.CarbonDataLoadSchema.DimensionRelation;
import org.pentaho.di.trans.steps.textfileinput.TextFileInputField;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/BlockDetails.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/BlockDetails.java b/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/BlockDetails.java
index cc6ae5f..67bbd51 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/BlockDetails.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/BlockDetails.java
@@ -21,7 +21,7 @@ package org.apache.carbondata.processing.csvreaderstep;
import java.io.Serializable;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/UnivocityCsvParser.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/UnivocityCsvParser.java b/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/UnivocityCsvParser.java
index 486c416..cbe6231 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/UnivocityCsvParser.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/UnivocityCsvParser.java
@@ -27,8 +27,8 @@ import java.io.Reader;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
-import org.apache.carbondata.core.datastorage.impl.FileFactory.FileType;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory.FileType;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java b/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java
index 7abe631..dd8ac61 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java
@@ -30,9 +30,6 @@ import java.util.Map;
import org.apache.carbondata.core.cache.Cache;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
-import org.apache.carbondata.core.metadata.Encoding;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.devapi.BiDictionary;
import org.apache.carbondata.core.devapi.DictionaryGenerationException;
@@ -41,6 +38,9 @@ import org.apache.carbondata.core.dictionary.generator.key.DictionaryKey;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.processing.newflow.dictionary.DictionaryServerClientDictionary;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java b/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java
index d26bf12..1a88a6c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java
@@ -37,21 +37,21 @@ import java.util.Set;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.processing.model.CarbonDataLoadSchema;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.processing.csvreaderstep.BlockDetails;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.processing.api.dataloader.DataLoadModel;
import org.apache.carbondata.processing.api.dataloader.SchemaInfo;
+import org.apache.carbondata.processing.csvreaderstep.BlockDetails;
import org.apache.carbondata.processing.csvreaderstep.CsvInputMeta;
import org.apache.carbondata.processing.graphgenerator.configuration.GraphConfigurationInfo;
import org.apache.carbondata.processing.mdkeygen.MDKeyGenStepMeta;
import org.apache.carbondata.processing.merger.step.CarbonSliceMergerStepMeta;
+import org.apache.carbondata.processing.model.CarbonDataLoadSchema;
import org.apache.carbondata.processing.schema.metadata.TableOptionWrapper;
import org.apache.carbondata.processing.sortandgroupby.sortdatastep.SortKeyStepMeta;
import org.apache.carbondata.processing.surrogatekeysgenerator.csvbased.CarbonCSVBasedSeqGenMeta;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStep.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStep.java b/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStep.java
index ac7c370..1fd84a4 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStep.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStep.java
@@ -32,16 +32,19 @@ import java.util.Map.Entry;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.common.logging.impl.StandardLogService;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.util.*;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.processing.datatypes.GenericDataType;
import org.apache.carbondata.processing.mdkeygen.file.FileData;
import org.apache.carbondata.processing.mdkeygen.file.FileManager;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
index 7a21b9e..45cac0a 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
@@ -27,8 +27,8 @@ import java.util.HashMap;
import java.util.List;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.updatestatus.LoadMetadataDetails;
import org.apache.carbondata.core.update.SegmentUpdateDetails;
+import org.apache.carbondata.core.updatestatus.LoadMetadataDetails;
import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager;
public class CarbonLoadModel implements Serializable {
@@ -43,8 +43,6 @@ public class CarbonLoadModel implements Serializable {
private String factFilePath;
- private String dimFolderPath;
-
private String colDictFilePath;
private String partitionId;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
index 8099427..f654926 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
@@ -24,13 +24,13 @@ import java.util.Map;
import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.processing.model.CarbonLoadModel;
import org.apache.carbondata.processing.newflow.constants.DataLoadProcessorConstants;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java
index 0a9e866..e81a962 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java
@@ -27,13 +27,13 @@ import java.util.Map;
import org.apache.carbondata.core.cache.Cache;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.devapi.BiDictionary;
import org.apache.carbondata.core.devapi.DictionaryGenerationException;
import org.apache.carbondata.core.dictionary.client.DictionaryClient;
import org.apache.carbondata.core.dictionary.generator.key.DictionaryKey;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.processing.newflow.DataField;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DirectDictionaryFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DirectDictionaryFieldConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DirectDictionaryFieldConverterImpl.java
index f40630b..7ae1b37 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DirectDictionaryFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DirectDictionaryFieldConverterImpl.java
@@ -21,9 +21,9 @@ package org.apache.carbondata.processing.newflow.converter.impl;
import java.util.List;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.processing.newflow.DataField;
import org.apache.carbondata.processing.newflow.converter.BadRecordLogHolder;
import org.apache.carbondata.processing.newflow.row.CarbonRow;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/FieldEncoderFactory.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/FieldEncoderFactory.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/FieldEncoderFactory.java
index 8dcff97..b89e8a4 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/FieldEncoderFactory.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/FieldEncoderFactory.java
@@ -24,11 +24,11 @@ import java.util.List;
import org.apache.carbondata.core.cache.Cache;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
+import org.apache.carbondata.core.dictionary.client.DictionaryClient;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.dictionary.client.DictionaryClient;
import org.apache.carbondata.processing.datatypes.ArrayDataType;
import org.apache.carbondata.processing.datatypes.GenericDataType;
import org.apache.carbondata.processing.datatypes.PrimitiveDataType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/MeasureFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/MeasureFieldConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/MeasureFieldConverterImpl.java
index beb1363..56e973b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/MeasureFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/MeasureFieldConverterImpl.java
@@ -20,9 +20,9 @@ package org.apache.carbondata.processing.newflow.converter.impl;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.processing.newflow.DataField;
import org.apache.carbondata.processing.newflow.converter.BadRecordLogHolder;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/NonDictionaryFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/NonDictionaryFieldConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/NonDictionaryFieldConverterImpl.java
index 99c41af..b0d9bb1 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/NonDictionaryFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/NonDictionaryFieldConverterImpl.java
@@ -20,9 +20,9 @@ package org.apache.carbondata.processing.newflow.converter.impl;
import java.nio.charset.Charset;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.processing.newflow.DataField;
import org.apache.carbondata.processing.newflow.converter.BadRecordLogHolder;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
index 3893ab8..4699542 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/impl/ParallelReadMergeSorterWithBucketingImpl.java
@@ -28,8 +28,8 @@ import java.util.concurrent.TimeUnit;
import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.metadata.schema.BucketingInfo;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.schema.BucketingInfo;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
import org.apache.carbondata.processing.newflow.DataField;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
index ff906fe..f4c60b2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
@@ -24,8 +24,8 @@ import java.io.IOException;
import java.util.Iterator;
import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.processing.constants.LoggerAction;
import org.apache.carbondata.processing.newflow.AbstractDataLoadProcessorStep;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
index 00fad6d..dac1390 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
@@ -26,12 +26,10 @@ import java.util.Iterator;
import java.util.List;
import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.schema.BucketingInfo;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.processing.newflow.steps.partition.Partitioner;
-import org.apache.carbondata.processing.newflow.steps.partition.impl.HashPartitionerImpl;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.processing.constants.LoggerAction;
import org.apache.carbondata.processing.newflow.AbstractDataLoadProcessorStep;
@@ -42,6 +40,8 @@ import org.apache.carbondata.processing.newflow.converter.RowConverter;
import org.apache.carbondata.processing.newflow.converter.impl.RowConverterImpl;
import org.apache.carbondata.processing.newflow.row.CarbonRow;
import org.apache.carbondata.processing.newflow.row.CarbonRowBatch;
+import org.apache.carbondata.processing.newflow.steps.partition.Partitioner;
+import org.apache.carbondata.processing.newflow.steps.partition.impl.HashPartitionerImpl;
import org.apache.carbondata.processing.surrogatekeysgenerator.csvbased.BadRecordsLogger;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java
index 131d7c0..8b996bd 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java
@@ -24,10 +24,10 @@ import java.util.Iterator;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.constants.IgnoreDictionary;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
import org.apache.carbondata.processing.newflow.AbstractDataLoadProcessorStep;
import org.apache.carbondata.processing.newflow.CarbonDataLoadConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/CompressedTempSortFileWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/CompressedTempSortFileWriter.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/CompressedTempSortFileWriter.java
index ab371fa..7423f03 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/CompressedTempSortFileWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/CompressedTempSortFileWriter.java
@@ -24,7 +24,7 @@ import java.io.DataOutputStream;
import java.io.IOException;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
index c6e76a4..63ce48a 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
@@ -22,8 +22,8 @@ import java.io.File;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.processing.newflow.CarbonDataLoadConfiguration;
import org.apache.carbondata.processing.schema.metadata.SortObserver;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
index c27190e..b62f0ab 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
@@ -41,23 +41,23 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.metadata.CarbonMetadata;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.columnar.BlockIndexerStorageForInt;
-import org.apache.carbondata.core.datastorage.columnar.BlockIndexerStorageForNoInvertedIndex;
-import org.apache.carbondata.core.datastorage.columnar.ColumnGroupModel;
-import org.apache.carbondata.core.datastorage.columnar.IndexStorage;
-import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForInt;
+import org.apache.carbondata.core.datastore.columnar.BlockIndexerStorageForNoInvertedIndex;
+import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
+import org.apache.carbondata.core.datastore.columnar.IndexStorage;
+import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.keygenerator.columnar.ColumnarSplitter;
import org.apache.carbondata.core.keygenerator.columnar.impl.MultiDimKeyVarLengthEquiSplitGenerator;
import org.apache.carbondata.core.keygenerator.factory.KeyGeneratorFactory;
+import org.apache.carbondata.core.metadata.CarbonMetadata;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index 801e023..06ae35e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -25,15 +25,15 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.metadata.CarbonMetadata;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.processing.datatypes.GenericDataType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/store/StoreFactory.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/StoreFactory.java b/processing/src/main/java/org/apache/carbondata/processing/store/StoreFactory.java
index f1e4713..a88332e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/StoreFactory.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/StoreFactory.java
@@ -19,9 +19,9 @@
package org.apache.carbondata.processing.store;
-import org.apache.carbondata.core.datastorage.NodeMeasureDataStore;
-import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastorage.impl.data.compressed.HeavyCompressedDoubleArrayDataInMemoryStore;
+import org.apache.carbondata.core.datastore.NodeMeasureDataStore;
+import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastore.impl.data.compressed.HeavyCompressedDoubleArrayDataInMemoryStore;
public final class StoreFactory {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/store/colgroup/ColGroupBlockStorage.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/colgroup/ColGroupBlockStorage.java b/processing/src/main/java/org/apache/carbondata/processing/store/colgroup/ColGroupBlockStorage.java
index 59383c4..5d9792e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/colgroup/ColGroupBlockStorage.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/colgroup/ColGroupBlockStorage.java
@@ -20,7 +20,7 @@ package org.apache.carbondata.processing.store.colgroup;
import java.util.concurrent.Callable;
-import org.apache.carbondata.core.datastorage.columnar.IndexStorage;
+import org.apache.carbondata.core.datastore.columnar.IndexStorage;
/**
* it is holder of column group data and also min max for colgroup block data
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index 2ef48af..2273d65 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -40,6 +40,13 @@ import java.util.concurrent.TimeUnit;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.columnar.IndexStorage;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
+import org.apache.carbondata.core.metadata.BlockletInfoColumnar;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.blocklet.index.BlockletBTreeIndex;
import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
@@ -51,13 +58,6 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.columnar.IndexStorage;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
-import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
-import org.apache.carbondata.core.metadata.BlockletInfoColumnar;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonMergerUtil;
import org.apache.carbondata.core.util.CarbonMetadataUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java
index 4a45b28..daef702 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java
@@ -19,8 +19,8 @@
package org.apache.carbondata.processing.store.writer;
-import org.apache.carbondata.core.datastorage.columnar.IndexStorage;
-import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastore.columnar.IndexStorage;
+import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
import org.apache.carbondata.processing.store.writer.exception.CarbonDataWriterException;
public interface CarbonFactDataWriter<T> {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/store/writer/NodeHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/NodeHolder.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/NodeHolder.java
index 885ecb8..9c33825 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/NodeHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/NodeHolder.java
@@ -21,7 +21,7 @@ package org.apache.carbondata.processing.store.writer;
import java.util.BitSet;
-import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
public class NodeHolder {
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
index 24435e8..a966739 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v1/CarbonFactDataWriterImplV1.java
@@ -27,8 +27,8 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.columnar.IndexStorage;
-import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastore.columnar.IndexStorage;
+import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
import org.apache.carbondata.core.metadata.BlockletInfoColumnar;
import org.apache.carbondata.core.util.CarbonMetadataUtil;
import org.apache.carbondata.core.writer.CarbonFooterWriter;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
index 7b93d2c..fe53adf 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
@@ -26,9 +26,9 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.BlockletInfoColumnar;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.util.CarbonMetadataUtil;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/BadRecordsLogger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/BadRecordsLogger.java b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/BadRecordsLogger.java
index 33430e7..8b530d3 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/BadRecordsLogger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/BadRecordsLogger.java
@@ -32,8 +32,8 @@ import java.util.Map;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
-import org.apache.carbondata.core.datastorage.impl.FileFactory.FileType;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory.FileType;
import org.apache.carbondata.core.util.CarbonUtil;
public class BadRecordsLogger {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
index 90ecaf2..4830576 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
@@ -32,10 +32,10 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.writer.HierarchyValueWriterForCSV;
import org.apache.carbondata.processing.datatypes.GenericDataType;
import org.apache.carbondata.processing.mdkeygen.file.IFileManagerComposite;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java
index b5d196f..dfd03b5 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java
@@ -28,8 +28,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.processing.datatypes.ArrayDataType;
import org.apache.carbondata.processing.datatypes.GenericDataType;
import org.apache.carbondata.processing.datatypes.PrimitiveDataType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
index 17c8441..75dddd2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
@@ -46,16 +46,16 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.common.logging.impl.StandardLogService;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
-import org.apache.carbondata.core.metadata.CarbonMetadata;
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.keygenerator.factory.KeyGeneratorFactory;
+import org.apache.carbondata.core.metadata.CarbonMetadata;
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -74,7 +74,6 @@ import org.apache.carbondata.processing.schema.metadata.ColumnsInfo;
import org.apache.carbondata.processing.schema.metadata.HierarchiesInfo;
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
import org.apache.carbondata.processing.util.RemoveDictionaryUtil;
-
import static org.apache.carbondata.processing.constants.TableOptionConstant.BAD_RECORDS_ACTION;
import static org.apache.carbondata.processing.constants.TableOptionConstant.BAD_RECORDS_LOGGER_ENABLE;
import static org.apache.carbondata.processing.constants.TableOptionConstant.SERIALIZATION_NULL_FORMAT;
@@ -90,6 +89,7 @@ import org.pentaho.di.trans.step.StepDataInterface;
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.trans.step.StepMetaInterface;
+
public class CarbonCSVBasedSeqGenStep extends BaseStep {
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
index e7b9a4c..835a3bf 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
@@ -33,13 +33,13 @@ import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.cache.CacheType;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
-import org.apache.carbondata.core.metadata.ColumnIdentifier;
-import org.apache.carbondata.core.metadata.CarbonMetadata;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
+import org.apache.carbondata.core.metadata.CarbonMetadata;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
import org.apache.carbondata.core.writer.ByteArrayHolder;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index 978ce56..68f3b60 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -36,7 +36,11 @@ import java.util.Set;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.processing.model.CarbonDataLoadSchema;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory.FileType;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
@@ -44,11 +48,6 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
-import org.apache.carbondata.core.datastorage.impl.FileFactory.FileType;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
@@ -56,6 +55,7 @@ import org.apache.carbondata.processing.datatypes.ArrayDataType;
import org.apache.carbondata.processing.datatypes.GenericDataType;
import org.apache.carbondata.processing.datatypes.PrimitiveDataType;
import org.apache.carbondata.processing.datatypes.StructDataType;
+import org.apache.carbondata.processing.model.CarbonDataLoadSchema;
import org.apache.carbondata.processing.newflow.DataField;
import org.apache.commons.lang3.ArrayUtils;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/main/java/org/apache/carbondata/processing/util/CarbonSchemaParser.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonSchemaParser.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonSchemaParser.java
index 9b58a84..c9304c0 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonSchemaParser.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonSchemaParser.java
@@ -30,16 +30,16 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-import org.apache.carbondata.processing.model.CarbonDataLoadSchema;
-import org.apache.carbondata.processing.model.CarbonDataLoadSchema.DimensionRelation;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.keygenerator.factory.KeyGeneratorFactory;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.keygenerator.factory.KeyGeneratorFactory;
import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.processing.model.CarbonDataLoadSchema;
+import org.apache.carbondata.processing.model.CarbonDataLoadSchema.DimensionRelation;
import org.apache.carbondata.processing.schema.metadata.ColumnSchemaDetails;
import org.apache.carbondata.processing.schema.metadata.ColumnSchemaDetailsWrapper;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java b/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
index 59fdda8..b0a89dd 100644
--- a/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
+++ b/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
@@ -45,7 +45,7 @@ import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.processing.csvreaderstep.BlockDetails;
import org.apache.carbondata.core.updatestatus.LoadMetadataDetails;
import org.apache.carbondata.core.util.CarbonProperties;
[05/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scan/complextypes/PrimitiveQueryTypeTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/complextypes/PrimitiveQueryTypeTest.java b/core/src/test/java/org/apache/carbondata/scan/complextypes/PrimitiveQueryTypeTest.java
deleted file mode 100644
index 4bfe5a5..0000000
--- a/core/src/test/java/org/apache/carbondata/scan/complextypes/PrimitiveQueryTypeTest.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.complextypes;
-
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.core.cache.dictionary.ColumnDictionaryInfo;
-import org.apache.carbondata.core.cache.dictionary.Dictionary;
-import org.apache.carbondata.core.cache.dictionary.ForwardDictionary;
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
-import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
-import org.apache.carbondata.core.keygenerator.mdkey.Bits;
-import org.apache.carbondata.core.util.DataTypeUtil;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.apache.spark.sql.types.BooleanType$;
-import org.apache.spark.sql.types.DoubleType$;
-import org.apache.spark.sql.types.IntegerType$;
-import org.apache.spark.sql.types.LongType$;
-import org.apache.spark.sql.types.TimestampType$;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-public class PrimitiveQueryTypeTest {
- private static PrimitiveQueryType primitiveQueryType, primitiveQueryTypeForInt,
- primitiveQueryTypeForLong, primitiveQueryTypeForDouble, primitiveQueryTypeForBoolean,
- primitiveQueryTypeForTimeStamp, primitiveQueryTypeForTimeStampForIsDictionaryFalse;
- private static Dictionary dictionary;
- private boolean isDirectDictionary = false;
-
- @BeforeClass public static void setUp() {
- String name = "test";
- String parentName = "testParent";
- int blockIndex = 1;
- int keySize = 1;
- boolean isDirectDictionary = true;
- primitiveQueryType =
- new PrimitiveQueryType(name, parentName, blockIndex, DataType.STRING, keySize, dictionary,
- isDirectDictionary);
- primitiveQueryTypeForInt =
- new PrimitiveQueryType(name, parentName, blockIndex, DataType.INT, keySize, dictionary,
- isDirectDictionary);
- primitiveQueryTypeForDouble =
- new PrimitiveQueryType(name, parentName, blockIndex, DataType.DOUBLE, keySize, dictionary,
- isDirectDictionary);
- primitiveQueryTypeForLong =
- new PrimitiveQueryType(name, parentName, blockIndex, DataType.LONG, keySize, dictionary,
- isDirectDictionary);
- primitiveQueryTypeForBoolean =
- new PrimitiveQueryType(name, parentName, blockIndex, DataType.BOOLEAN, keySize, dictionary,
- isDirectDictionary);
- primitiveQueryTypeForTimeStamp =
- new PrimitiveQueryType(name, parentName, blockIndex, DataType.TIMESTAMP, keySize,
- dictionary, isDirectDictionary);
- ColumnDictionaryInfo columnDictionaryInfo = new ColumnDictionaryInfo(DataType.STRING);
- ForwardDictionary forwardDictionary = new ForwardDictionary(columnDictionaryInfo);
- primitiveQueryTypeForTimeStampForIsDictionaryFalse =
- new PrimitiveQueryType(name, parentName, blockIndex, DataType.TIMESTAMP, keySize,
- forwardDictionary, false);
-
- }
-
- @Test public void testGetDataBasedOnDataTypeFromSurrogates() {
- ByteBuffer surrogateData = ByteBuffer.allocate(10);
- surrogateData.put(3, (byte) 1);
- new MockUp<Bits>() {
- @Mock public long[] getKeyArray(byte[] key, int offset) {
- return new long[] { 1313045L };
- }
- };
- Object expectedValue = 1313043000000L;
-
- Object actualValue =
- primitiveQueryTypeForTimeStamp.getDataBasedOnDataTypeFromSurrogates(surrogateData);
- assertEquals(expectedValue, actualValue);
- }
-
- @Test public void testGetDataBasedOnDataTypeFromSurrogatesWhenIsDictionaryFalse() {
- ByteBuffer surrogateData = ByteBuffer.allocate(10);
- surrogateData.put(3, (byte) 1);
- new MockUp<ForwardDictionary>() {
- @Mock public String getDictionaryValueForKey(int surrogateKey) {
- return "2015-10-20 12:30:01";
- }
- };
-
- Object expectedValue = primitiveQueryTypeForTimeStampForIsDictionaryFalse
- .getDataBasedOnDataTypeFromSurrogates(surrogateData);
- Object actualValue = primitiveQueryTypeForTimeStampForIsDictionaryFalse
- .getDataBasedOnDataTypeFromSurrogates(surrogateData);
- assertEquals(expectedValue, actualValue);
- }
-
- public Object getDataBasedOnDataTypeFromSurrogates(ByteBuffer surrogateData) {
- int keySize = 2;
- byte[] data = new byte[keySize];
- surrogateData.get(data);
- Bits bit = new Bits(new int[] { keySize * 8 });
- int surrgateValue = (int) bit.getKeyArray(data, 0)[0];
- Object actualData = null;
- if (isDirectDictionary) {
- DirectDictionaryGenerator directDictionaryGenerator =
- DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(DataType.TIMESTAMP);
- actualData = directDictionaryGenerator.getValueFromSurrogate(surrgateValue);
- } else {
- String dictionaryValueForKey = dictionary.getDictionaryValueForKey(surrgateValue);
- actualData = DataTypeUtil.getDataBasedOnDataType(dictionaryValueForKey, DataType.TIMESTAMP);
- }
- return actualData;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scan/complextypes/StructQueryTypeTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/complextypes/StructQueryTypeTest.java b/core/src/test/java/org/apache/carbondata/scan/complextypes/StructQueryTypeTest.java
deleted file mode 100644
index 5d2b2cf..0000000
--- a/core/src/test/java/org/apache/carbondata/scan/complextypes/StructQueryTypeTest.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.complextypes;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-
-public class StructQueryTypeTest {
- private static StructQueryType structQueryType;
- private static ArrayQueryType arrayQueryType;
-
- @BeforeClass public static void setUp() {
- String name = "testName";
- String parentName = "testName";
- int blockIndex = 5;
- structQueryType = new StructQueryType(name, parentName, blockIndex);
- arrayQueryType = new ArrayQueryType(name, parentName, blockIndex);
- }
-
- @Test public void testGetDataBasedOnDataTypeFromSurrogates() {
- ByteBuffer surrogateData = ByteBuffer.allocate(10);
- surrogateData.put(3, (byte) 1);
- structQueryType.addChildren(arrayQueryType);
- List children = new ArrayList();
- children.add(arrayQueryType);
- assertNotNull(structQueryType.getDataBasedOnDataTypeFromSurrogates(surrogateData));
- }
-
- @Test public void testGetColsCount() {
- structQueryType.setName("testName");
- structQueryType.setParentname("testName");
- structQueryType.addChildren(arrayQueryType);
- new MockUp<ArrayQueryType>() {
- @Mock int getColsCount() {
- return 1;
- }
- };
- int actualValue = structQueryType.getColsCount();
- int expectedValue = 2;
- assertEquals(expectedValue, actualValue);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scan/executor/util/QueryUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/executor/util/QueryUtilTest.java b/core/src/test/java/org/apache/carbondata/scan/executor/util/QueryUtilTest.java
deleted file mode 100644
index f127079..0000000
--- a/core/src/test/java/org/apache/carbondata/scan/executor/util/QueryUtilTest.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.executor.util;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.block.SegmentPropertiesTestUtil;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.scan.model.QueryDimension;
-
-import junit.framework.TestCase;
-import mockit.Mock;
-import mockit.MockUp;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static org.junit.Assert.assertArrayEquals;
-
-public class QueryUtilTest extends TestCase {
-
- private SegmentProperties segmentProperties;
-
- @BeforeClass public void setUp() {
- segmentProperties = SegmentPropertiesTestUtil.getSegmentProperties();
- }
-
- @Test public void testGetMaskedByteRangeGivingProperMaksedByteRange() {
-
- QueryDimension dimension =
- new QueryDimension(segmentProperties.getDimensions().get(0).getColName());
- dimension.setDimension(segmentProperties.getDimensions().get(0));
- int[] maskedByteRange = QueryUtil
- .getMaskedByteRange(Arrays.asList(dimension), segmentProperties.getDimensionKeyGenerator());
- int[] expectedMaskedByteRange = { 0 };
- for (int i = 0; i < maskedByteRange.length; i++) {
- assertEquals(expectedMaskedByteRange[i], maskedByteRange[i]);
- }
- }
-
- @Test public void testGetMaskedByteRangeGivingProperMaksedByteRangeOnlyForDictionaryKey() {
- List<QueryDimension> dimensions = new ArrayList<QueryDimension>();
- for (int i = 0; i < 2; i++) {
- QueryDimension dimension =
- new QueryDimension(segmentProperties.getDimensions().get(i).getColName());
- dimension.setDimension(segmentProperties.getDimensions().get(i));
- dimensions.add(dimension);
- }
- int[] maskedByteRange =
- QueryUtil.getMaskedByteRange(dimensions, segmentProperties.getDimensionKeyGenerator());
- int[] expectedMaskedByteRange = { 0 };
- for (int i = 0; i < maskedByteRange.length; i++) {
- assertEquals(expectedMaskedByteRange[i], maskedByteRange[i]);
- }
- }
-
- @Test public void testGetMaskedByteRangeBasedOrdinalGivingProperMaskedByte() {
- List<Integer> dimensionOrdinal = new ArrayList<Integer>();
- dimensionOrdinal.add(0);
- int[] maskedByteRange = QueryUtil.getMaskedByteRangeBasedOrdinal(dimensionOrdinal,
- segmentProperties.getDimensionKeyGenerator());
- int[] expectedMaskedByteRange = { 0 };
- for (int i = 0; i < maskedByteRange.length; i++) {
- assertEquals(expectedMaskedByteRange[i], maskedByteRange[i]);
- }
- }
-
- @Test public void testGetMaxKeyBasedOnDimensions() {
- List<QueryDimension> dimensions = new ArrayList<QueryDimension>();
- for (int i = 0; i < 2; i++) {
- QueryDimension dimension =
- new QueryDimension(segmentProperties.getDimensions().get(i).getColName());
- dimension.setDimension(segmentProperties.getDimensions().get(i));
- dimensions.add(dimension);
- }
- byte[] maxKeyBasedOnDimensions = null;
- try {
- maxKeyBasedOnDimensions = QueryUtil
- .getMaxKeyBasedOnDimensions(dimensions, segmentProperties.getDimensionKeyGenerator());
- } catch (KeyGenException e) {
- assertTrue(false);
- }
- byte[] expectedMaxKeyBasedOnDimensions = { -1, 0, 0, 0, 0, 0 };
- for (int i = 0; i < expectedMaxKeyBasedOnDimensions.length; i++) {
- if (expectedMaxKeyBasedOnDimensions[i] != maxKeyBasedOnDimensions[i]) {
- assertTrue(false);
- }
- }
- long[] expectedKeyArray = { 255, 0, 0, 0, 0, 0 };
- long[] keyArray =
- segmentProperties.getDimensionKeyGenerator().getKeyArray(maxKeyBasedOnDimensions);
- for (int i = 0; i < keyArray.length; i++) {
- if (expectedKeyArray[i] != keyArray[i]) {
- assertTrue(false);
- }
- }
- }
-
- @Test public void testGetMaksedByte() {
- QueryDimension dimension =
- new QueryDimension(segmentProperties.getDimensions().get(0).getColName());
- dimension.setDimension(segmentProperties.getDimensions().get(0));
- dimension.setDimension(segmentProperties.getDimensions().get(0));
- int[] maskedByteRange = QueryUtil
- .getMaskedByteRange(Arrays.asList(dimension), segmentProperties.getDimensionKeyGenerator());
- int[] maskedByte = QueryUtil
- .getMaskedByte(segmentProperties.getDimensionKeyGenerator().getDimCount(), maskedByteRange);
- int[] expectedMaskedByte = { 0, -1, -1, -1, -1, -1 };
-
- for (int i = 0; i < expectedMaskedByte.length; i++) {
- if (expectedMaskedByte[i] != maskedByte[i]) {
- assertTrue(false);
- }
- }
- }
-
- @Test public void testSearchInArrayWithSearchInputNotPresentInArray() {
- int[] dummyArray = { 1, 2, 3, 4, 5 };
- int searchInput = 6;
- boolean result = QueryUtil.searchInArray(dummyArray, searchInput);
- assert (!result);
- }
-
- @Test public void testSearchInArrayWithSearchInputPresentInArray() {
- int[] dummyArray = { 1, 2, 3, 4, 5 };
- int searchInput = 1;
- boolean result = QueryUtil.searchInArray(dummyArray, searchInput);
- assert (result);
- }
-
- @Test public void testGetColumnGroupIdWhenOrdinalValueNotPresentInArrayIndex() {
- int ordinal = 0;
- new MockUp<SegmentProperties>() {
- @Mock public int[][] getColumnGroups() {
- int columnGroups[][] = { { 1, 1 }, { 2, 2 }, { 3, 3 }, { 4, 4 }, { 5, 5 } };
- return columnGroups;
- }
- };
- int actualValue = QueryUtil.getColumnGroupId(segmentProperties, ordinal);
- int expectedValue = 4; //expectedValue will always be arrayLength - 1
- assertEquals(expectedValue, actualValue);
- }
-
- @Test public void testGetColumnGroupIdWhenOrdinalValuePresentInArrayIndex() {
- int ordinal = 1;
- new MockUp<SegmentProperties>() {
- @Mock public int[][] getColumnGroups() {
- int columnGroups[][] = { { 1, 1 }, { 2, 2 }, { 3, 3 }, { 4, 4 }, { 5, 5 } };
- return columnGroups;
- }
- };
- int actualValue = QueryUtil.getColumnGroupId(segmentProperties, ordinal);
- int expectedValue = 0;
- assertEquals(expectedValue, actualValue);
- }
-
- @Test public void testGetColumnGroupIdWhenColumnGroupsIndexValueLengthLessThanOne() {
- int ordinal = 1;
- new MockUp<SegmentProperties>() {
- @Mock public int[][] getColumnGroups() {
- int columnGroups[][] = { { 1 } };
- return columnGroups;
- }
- };
- int actualValue = QueryUtil.getColumnGroupId(segmentProperties, ordinal);
- int expectedValue = -1;
- assertEquals(expectedValue, actualValue);
- }
-
- @Test public void testGetMaskedKey() {
- byte[] data = { 1, 2, 3, 4, 5, 5 };
- byte[] maxKey = { 15, 20, 25, 30, 35, 35 };
- int[] maskByteRanges = { 1, 2, 3, 4, 5 };
- int byteCount = 5;
- byte[] actualValue = QueryUtil.getMaskedKey(data, maxKey, maskByteRanges, byteCount);
- byte[] expectedValue = { 0, 1, 4, 1, 1 };
- assertArrayEquals(expectedValue, actualValue);
- }
-
- @Test public void testGetMaxKeyBasedOnOrinal() throws Exception {
- List<Integer> dummyList = new ArrayList<>();
- dummyList.add(0, 1);
- dummyList.add(1, 2);
- byte[] actualValue =
- QueryUtil.getMaxKeyBasedOnOrinal(dummyList, segmentProperties.getDimensionKeyGenerator());
- byte[] expectedValue = { 0, -1, -1, 0, 0, 0 };
- assertArrayEquals(expectedValue, actualValue);
- }
-
- @Test public void testGetSortDimensionIndexes() {
- List<QueryDimension> sortedDimensions = new ArrayList<QueryDimension>();
- for (int i = 0; i < 2; i++) {
- QueryDimension dimension =
- new QueryDimension(segmentProperties.getDimensions().get(i).getColName());
- dimension.setDimension(segmentProperties.getDimensions().get(i));
- sortedDimensions.add(dimension);
- }
- List<QueryDimension> queryDimensions = new ArrayList<QueryDimension>();
- for (int i = 0; i < 2; i++) {
- QueryDimension dimension =
- new QueryDimension(segmentProperties.getDimensions().get(i).getColName());
- dimension.setDimension(segmentProperties.getDimensions().get(i));
- queryDimensions.add(dimension);
- }
- byte[] actualValue = QueryUtil.getSortDimensionIndexes(sortedDimensions, queryDimensions);
- byte[] expectedValue = { 0, 0 };
- assertArrayEquals(expectedValue, actualValue);
- }
-
- @Test public void testGetActualTypeIndex() {
- List<String> dummyList = new ArrayList<>();
- dummyList.add("test1");
- dummyList.add("test2");
- int[] actualValue = QueryUtil.getActualTypeIndex(dummyList);
- int[] expectedValue = { 0, 1 };
- assertArrayEquals(expectedValue, actualValue);
- }
-
- @AfterClass public void tearDown() {
- segmentProperties = null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scan/executor/util/RestructureUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/executor/util/RestructureUtilTest.java b/core/src/test/java/org/apache/carbondata/scan/executor/util/RestructureUtilTest.java
deleted file mode 100644
index dbe12be..0000000
--- a/core/src/test/java/org/apache/carbondata/scan/executor/util/RestructureUtilTest.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.executor.util;
-
-import org.apache.carbondata.core.metadata.Encoding;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.scan.executor.infos.AggregatorInfo;
-import org.apache.carbondata.scan.model.QueryDimension;
-import org.apache.carbondata.scan.model.QueryMeasure;
-
-import org.junit.Test;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.MatcherAssert.assertThat;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-public class RestructureUtilTest {
-
- @Test public void testToGetUpdatedQueryDimension() {
- List<Encoding> encodingList = new ArrayList<Encoding>();
- encodingList.add(Encoding.DICTIONARY);
- ColumnSchema columnSchema1 = new ColumnSchema();
- columnSchema1.setColumnName("Id");
- columnSchema1.setEncodingList(encodingList);
- ColumnSchema columnSchema2 = new ColumnSchema();
- columnSchema2.setColumnName("Name");
- columnSchema2.setEncodingList(encodingList);
- ColumnSchema columnSchema3 = new ColumnSchema();
- columnSchema3.setColumnName("Age");
- columnSchema3.setEncodingList(encodingList);
- ColumnSchema columnSchema4 = new ColumnSchema();
- columnSchema4.setColumnName("Salary");
- columnSchema4.setEncodingList(encodingList);
- ColumnSchema columnSchema5 = new ColumnSchema();
- columnSchema5.setColumnName("Address");
- columnSchema5.setEncodingList(encodingList);
-
- CarbonDimension tableBlockDimension1 = new CarbonDimension(columnSchema1, 1, 1, 1, 1);
- CarbonDimension tableBlockDimension2 = new CarbonDimension(columnSchema2, 5, 5, 5, 5);
- List<CarbonDimension> tableBlockDimensions =
- Arrays.asList(tableBlockDimension1, tableBlockDimension2);
-
- CarbonDimension tableComplexDimension1 = new CarbonDimension(columnSchema3, 4, 4, 4, 4);
- CarbonDimension tableComplexDimension2 = new CarbonDimension(columnSchema4, 2, 2, 2, 2);
- List<CarbonDimension> tableComplexDimensions =
- Arrays.asList(tableComplexDimension1, tableComplexDimension2);
-
- QueryDimension queryDimension1 = new QueryDimension("Id");
- queryDimension1.setDimension(tableBlockDimension1);
- QueryDimension queryDimension2 = new QueryDimension("Name");
- queryDimension2.setDimension(tableComplexDimension2);
- QueryDimension queryDimension3 = new QueryDimension("Address");
- queryDimension3.setDimension(new CarbonDimension(columnSchema5, 3, 3, 3, 3));
-
- List<QueryDimension> queryDimensions =
- Arrays.asList(queryDimension1, queryDimension2, queryDimension3);
-
- List<QueryDimension> result = RestructureUtil
- .getUpdatedQueryDimension(queryDimensions, tableBlockDimensions, tableComplexDimensions);
-
- assertThat(result, is(equalTo(Arrays.asList(queryDimension1, queryDimension2))));
- }
-
- @Test public void testToGetAggregatorInfos() {
- ColumnSchema columnSchema1 = new ColumnSchema();
- columnSchema1.setColumnName("Id");
- ColumnSchema columnSchema2 = new ColumnSchema();
- columnSchema2.setColumnName("Name");
- ColumnSchema columnSchema3 = new ColumnSchema();
- columnSchema3.setColumnName("Age");
-
- CarbonMeasure carbonMeasure1 = new CarbonMeasure(columnSchema1, 1);
- CarbonMeasure carbonMeasure2 = new CarbonMeasure(columnSchema2, 2);
- CarbonMeasure carbonMeasure3 = new CarbonMeasure(columnSchema3, 3);
- carbonMeasure3.setDefaultValue("3".getBytes());
- List<CarbonMeasure> currentBlockMeasures = Arrays.asList(carbonMeasure1, carbonMeasure2);
-
- QueryMeasure queryMeasure1 = new QueryMeasure("Id");
- queryMeasure1.setMeasure(carbonMeasure1);
- QueryMeasure queryMeasure2 = new QueryMeasure("Name");
- queryMeasure2.setMeasure(carbonMeasure2);
- QueryMeasure queryMeasure3 = new QueryMeasure("Age");
- queryMeasure3.setMeasure(carbonMeasure3);
- List<QueryMeasure> queryMeasures = Arrays.asList(queryMeasure1, queryMeasure2, queryMeasure3);
-
- AggregatorInfo aggregatorInfo =
- RestructureUtil.getAggregatorInfos(queryMeasures, currentBlockMeasures);
- boolean[] measuresExist = { true, true, false };
- assertThat(aggregatorInfo.getMeasureExists(), is(equalTo(measuresExist)));
- Object[] defaultValues = { null, null, "3".getBytes() };
- assertThat(aggregatorInfo.getDefaultValues(), is(equalTo(defaultValues)));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scan/expression/ColumnExpressionTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/expression/ColumnExpressionTest.java b/core/src/test/java/org/apache/carbondata/scan/expression/ColumnExpressionTest.java
deleted file mode 100644
index b8ed85e..0000000
--- a/core/src/test/java/org/apache/carbondata/scan/expression/ColumnExpressionTest.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.scan.filter.intf.RowImpl;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-public class ColumnExpressionTest {
-
- private ColumnExpression columnExpression;
-
- @Before public void setUp() {
- String columnName = "IMEI";
- DataType dataType = DataType.STRING;
- columnExpression = new ColumnExpression(columnName, dataType);
- }
-
- @Test public void testEvaluate() {
- RowImpl rowImpl = new RowImpl();
- rowImpl.setValues(new Integer[] { 1 });
- new MockUp<RowImpl>() {
- @Mock public Object getVal(int index) {
- return 1;
- }
- };
- ExpressionResult expectedValue = new ExpressionResult(DataType.INT, 1);
- assertEquals(expectedValue, columnExpression.evaluate(rowImpl));
- }
-
- @Test public void testEvaluateForNullValue() {
- RowImpl rowImpl = new RowImpl();
- rowImpl.setValues(null);
- new MockUp<RowImpl>() {
- @Mock public Object getVal(int index) {
- return null;
- }
- };
- ExpressionResult expectedValue = new ExpressionResult(null);
- assertEquals(expectedValue, columnExpression.evaluate(rowImpl));
- }
-
- @Test public void testGetString() {
- String actualValue = columnExpression.getString();
- String expectedValue = "ColumnExpression(IMEI)";
- assertEquals(expectedValue, actualValue);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scan/expression/ExpressionResultTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/expression/ExpressionResultTest.java b/core/src/test/java/org/apache/carbondata/scan/expression/ExpressionResultTest.java
deleted file mode 100644
index 1cad885..0000000
--- a/core/src/test/java/org/apache/carbondata/scan/expression/ExpressionResultTest.java
+++ /dev/null
@@ -1,498 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.expression;
-
-import java.math.BigDecimal;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static junit.framework.TestCase.assertTrue;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.core.Is.is;
-import static org.hamcrest.core.IsEqual.equalTo;
-import static org.junit.Assert.assertEquals;
-import static org.pentaho.di.core.util.Assert.assertFalse;
-import static org.pentaho.di.core.util.Assert.assertNull;
-
-public class ExpressionResultTest {
- private static final double DELTA = 1e-15;
- private static ExpressionResult expressionResult;
-
- @BeforeClass public static void setUp() {
- expressionResult = new ExpressionResult(DataType.INT, null);
- }
-
- @Test public void testGetIntForNull() throws FilterIllegalMemberException {
- assertNull(expressionResult.getInt());
- }
-
- @Test public void testGetIntForString() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForString = new ExpressionResult(DataType.STRING, "5");
- int actualValue = expressionResultForString.getInt();
- int expectedValue = 5;
- assertEquals(expectedValue, actualValue);
- }
-
- @Test(expected = FilterIllegalMemberException.class)
- public void testGetIntForNumberFormatException() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForString = new ExpressionResult(DataType.STRING, "FOOBAR");
- expressionResultForString.getInt();
- }
-
- @Test public void testGetIntForDouble() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForDouble = new ExpressionResult(DataType.DOUBLE, 5.0);
- int actualValue = expressionResultForDouble.getInt();
- int expectedValue = 5;
- assertEquals(expectedValue, actualValue);
- }
-
- @Test public void testGetIntForInt() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForInt = new ExpressionResult(DataType.INT, 5);
- int actualValue = expressionResultForInt.getInt();
- int expectedValue = 5;
- assertEquals(expectedValue, actualValue);
- }
-
- @Test(expected = FilterIllegalMemberException.class)
- public void testGetIntForIllegalMemberException() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForInt = new ExpressionResult(DataType.FLOAT, 25.36);
- expressionResultForInt.getInt();
- }
-
- @Test(expected = FilterIllegalMemberException.class) public void testGetIntForClassCastException()
- throws FilterIllegalMemberException {
- ExpressionResult expressionResultForInt = new ExpressionResult(DataType.INT, "STRING");
- expressionResultForInt.getInt();
- }
-
- @Test public void testGetShortForNull() throws FilterIllegalMemberException {
- assertNull(expressionResult.getShort());
- }
-
- @Test public void testGetShortForString() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForString = new ExpressionResult(DataType.STRING, "5");
- short actualValue = expressionResultForString.getShort();
- short expectedValue = 5;
- assertEquals(expectedValue, actualValue);
- }
-
- @Test(expected = FilterIllegalMemberException.class)
- public void testGetShortForNumberFormatException() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForString = new ExpressionResult(DataType.STRING, "FOOBAR");
- expressionResultForString.getShort();
- }
-
- @Test public void testGetShortForDouble() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForDouble = new ExpressionResult(DataType.DOUBLE, 5.0);
- short actualValue = expressionResultForDouble.getShort();
- short expectedValue = 5;
- assertEquals(expectedValue, actualValue);
- }
-
- @Test public void testGetShortForInt() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForInt = new ExpressionResult(DataType.INT, 5);
- short actualValue = expressionResultForInt.getShort();
- short expectedValue = 5;
- assertEquals(expectedValue, actualValue);
- }
-
- @Test(expected = FilterIllegalMemberException.class)
- public void testGetShortForIllegalMemberException() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForInt = new ExpressionResult(DataType.FLOAT, 25.36);
- expressionResultForInt.getShort();
- }
-
- @Test(expected = FilterIllegalMemberException.class)
- public void testGetShortForClassCastException() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForInt = new ExpressionResult(DataType.INT, "STRING");
- expressionResultForInt.getShort();
- }
-
- @Test public void testGetStringForNull() throws FilterIllegalMemberException {
- assertNull(expressionResult.getString());
- }
-
- @Test public void testGetStringForString() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForString = new ExpressionResult(DataType.STRING, "5");
- String actualValue = expressionResultForString.getString();
- String expectedValue = "5";
- assertEquals(expectedValue, actualValue);
- }
-
- @Test public void testGetStringForException() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForString = new ExpressionResult(DataType.INT, "5");
- String actualValue = expressionResultForString.getString();
- String expectedValue = "5";
- assertEquals(expectedValue, actualValue);
- }
-
- @Test public void testGetDoubleForNull() throws FilterIllegalMemberException {
- assertNull(expressionResult.getDouble());
- }
-
- @Test public void testGetDoubleForString() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForString = new ExpressionResult(DataType.STRING, "5");
- double actualValue = expressionResultForString.getDouble();
- double expectedValue = 5;
- assertEquals(expectedValue, actualValue, DELTA);
- }
-
- @Test(expected = FilterIllegalMemberException.class)
- public void testGetDoubleForNumberFormatException() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForString = new ExpressionResult(DataType.STRING, "FOOBAR");
- expressionResultForString.getDouble();
- }
-
- @Test public void testGetDoubleForInt() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForDouble = new ExpressionResult(DataType.INT, 5);
- double actualValue = expressionResultForDouble.getDouble();
- double expectedValue = 5.0;
- assertEquals(expectedValue, actualValue, DELTA);
- }
-
- @Test public void testGetDoubleForDouble() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForDouble = new ExpressionResult(DataType.DOUBLE, 5.0);
- double actualValue = expressionResultForDouble.getDouble();
- double expectedValue = 5;
- assertEquals(expectedValue, actualValue, DELTA);
- }
-
- @Test(expected = FilterIllegalMemberException.class)
- public void testGetDoubleForIllegalMemberException() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForDouble = new ExpressionResult(DataType.FLOAT, 25.36);
- expressionResultForDouble.getDouble();
- }
-
- @Test(expected = FilterIllegalMemberException.class)
- public void testGetDoubleForClassCastException() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForDouble = new ExpressionResult(DataType.DOUBLE, "STRING");
- expressionResultForDouble.getDouble();
- }
-
- @Test public void testGetLongForNull() throws FilterIllegalMemberException {
- assertNull(expressionResult.getLong());
- }
-
- @Test public void testGetLongForString() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForString = new ExpressionResult(DataType.STRING, "5");
- long actualValue = expressionResultForString.getLong();
- long expectedValue = 5;
- assertEquals(expectedValue, actualValue);
- }
-
- @Test(expected = FilterIllegalMemberException.class)
- public void testGetLongForNumberFormatException() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForString = new ExpressionResult(DataType.STRING, "FOOBAR");
- expressionResultForString.getLong();
- }
-
- @Test(expected = FilterIllegalMemberException.class)
- public void testGetLongForIllegalMemberException() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForLong = new ExpressionResult(DataType.FLOAT, 25.36);
- expressionResultForLong.getLong();
- }
-
- @Test(expected = FilterIllegalMemberException.class)
- public void testGetLongForClassCastException() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForLong = new ExpressionResult(DataType.LONG, "STRING");
- expressionResultForLong.getLong();
- }
-
- @Test public void testGetDecimalForNull() throws FilterIllegalMemberException {
- assertNull(expressionResult.getDecimal());
- }
-
- @Test public void testGetDecimalForString() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForString = new ExpressionResult(DataType.STRING, "5");
- BigDecimal actualValue = expressionResultForString.getDecimal();
- BigDecimal expectedValue = new BigDecimal(5.00);
- assertEquals(expectedValue, actualValue);
- }
-
- @Test(expected = FilterIllegalMemberException.class)
- public void testGetDecimalForNumberFormatException() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForString = new ExpressionResult(DataType.STRING, "FOOBAR");
- expressionResultForString.getDecimal();
- }
-
- @Test public void testGetDecimalForInt() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForInt = new ExpressionResult(DataType.INT, 5);
- BigDecimal actualValue = expressionResultForInt.getDecimal();
- BigDecimal expectedValue = new BigDecimal(5);
- assertEquals(expectedValue, actualValue);
- }
-
- @Test public void testGetDecimalForDouble() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForDouble = new ExpressionResult(DataType.DOUBLE, 5);
- BigDecimal actualValue = expressionResultForDouble.getDecimal();
- BigDecimal expectedValue = new BigDecimal(5);
- assertEquals(expectedValue, actualValue);
- }
-
- @Test public void testGetDecimalForDecimal() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForDecimal = new ExpressionResult(DataType.DECIMAL, 5);
- BigDecimal actualValue = expressionResultForDecimal.getDecimal();
- BigDecimal expectedValue = new BigDecimal(5);
- assertEquals(expectedValue, actualValue);
- }
-
- @Test(expected = FilterIllegalMemberException.class)
- public void testGetDecimalForIllegalMemberException() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForFloat = new ExpressionResult(DataType.FLOAT, 25.36);
- expressionResultForFloat.getDecimal();
- }
-
- @Test(expected = FilterIllegalMemberException.class)
- public void testGetDecimalForClassCastException() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForException = new ExpressionResult(DataType.LONG, "STRING");
- expressionResultForException.getDecimal();
- }
-
- @Test public void testGetTimeForNull() throws FilterIllegalMemberException {
- assertNull(expressionResult.getTime());
- }
-
- @Test public void testGetTimeForString() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForString =
- new ExpressionResult(DataType.STRING, "2016-11-07 10:15:09");
- long actualValue = expressionResultForString.getTime();
- long expectedValue = getTime("2016-11-07 10:15:09");
- assertEquals(expectedValue, actualValue);
- }
-
- public Long getTime(String value) throws FilterIllegalMemberException {
- if (value == null) {
- return null;
- }
- SimpleDateFormat parser =
- new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT);
- Date dateToStr;
- try {
- dateToStr = parser.parse(value.toString());
- return dateToStr.getTime();
- } catch (ParseException e) {
- throw new FilterIllegalMemberException("Cannot convert value to Time/Long type value");
- }
- }
-
- @Test(expected = FilterIllegalMemberException.class) public void testGetTimeForParseException()
- throws FilterIllegalMemberException {
- ExpressionResult expressionResultForParseException =
- new ExpressionResult(DataType.STRING, "FOOBAR");
- expressionResultForParseException.getTime();
- }
-
- @Test(expected = FilterIllegalMemberException.class)
- public void testGetTimeForIllegalMemberException() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForFloat = new ExpressionResult(DataType.FLOAT, 25.36);
- expressionResultForFloat.getTime();
- }
-
- @Test(expected = FilterIllegalMemberException.class)
- public void testGetTimeForClassCastException() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForException = new ExpressionResult(DataType.LONG, "STRING");
- expressionResultForException.getTime();
- }
-
- @Test public void testGetBooleanForNull() throws FilterIllegalMemberException {
- assertNull(expressionResult.getBoolean());
- }
-
- @Test public void testGetBooleanForString() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForString = new ExpressionResult(DataType.STRING, "true");
- boolean actualValue = expressionResultForString.getBoolean();
- boolean expectedValue = true;
- assertEquals(expectedValue, actualValue);
- }
-
- @Test public void testGetBooleanForBoolean() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForString = new ExpressionResult(DataType.BOOLEAN, "true");
- boolean actualValue = expressionResultForString.getBoolean();
- boolean expectedValue = true;
- assertEquals(expectedValue, actualValue);
- }
-
- @Test(expected = FilterIllegalMemberException.class)
- public void testGetBooleanForIllegalMemberException() throws FilterIllegalMemberException {
- ExpressionResult expressionResultForFloat = new ExpressionResult(DataType.FLOAT, 25.36);
- expressionResultForFloat.getBoolean();
- }
-
- @Test public void testGetList() {
- ExpressionResult obj = new ExpressionResult(DataType.STRING, "test");
- List<ExpressionResult> actualValue = obj.getList();
- List<ExpressionResult> expected = new ArrayList<>();
- expected.add(0, obj);
- assertEquals(expected, actualValue);
- }
-
- @Test public void testHashCode() {
- ExpressionResult obj = new ExpressionResult(DataType.INT, 1);
- int actualValue = obj.hashCode();
- int expectedValue = 80;
- assertEquals(expectedValue, actualValue);
- }
-
- @Test public void testHashCodeForNull() {
- ExpressionResult obj = new ExpressionResult(null);
- int actualValue = obj.hashCode();
- int expectedValue = 31;
- assertEquals(expectedValue, actualValue);
- }
-
- @Test public void testHashCodeForNul() {
- ExpressionResult obj = new ExpressionResult(DataType.DOUBLE, null);
- int actualValue = obj.hashCode();
- int expectedValue = 31;
- assertEquals(expectedValue, actualValue);
- }
-
- @Test public void testEqualsForObjNOtInstanceOfExpressionResult() {
- ExpressionResult expressionResult = new ExpressionResult(DataType.INT, 5);
- List obj = new ArrayList();
- obj.add(0, 0);
- obj.add(1, 1);
- boolean result = expressionResult.equals(obj);
- assertFalse(result);
- }
-
- @Test public void testEqualsForString() {
- ExpressionResult expressionResult = new ExpressionResult(DataType.STRING, "5");
- ExpressionResult objToCompare = new ExpressionResult(DataType.STRING, "6");
- boolean result = expressionResult.equals(objToCompare);
- assertFalse(result);
- }
-
- @Test public void testEqualsForShort() {
- ExpressionResult expressionResult = new ExpressionResult(DataType.SHORT, 5);
- ExpressionResult objToCompare = new ExpressionResult(DataType.SHORT, 6);
- boolean result = expressionResult.equals(objToCompare);
- assertFalse(result);
- }
-
- @Test public void testEqualsForInt() {
- ExpressionResult expressionResult = new ExpressionResult(DataType.INT, 5);
- ExpressionResult objToCompare = new ExpressionResult(DataType.INT, 6);
- boolean result = expressionResult.equals(objToCompare);
- assertFalse(result);
- }
-
- @Test public void testEqualsForDecimal() {
- ExpressionResult expressionResult = new ExpressionResult(DataType.DECIMAL, 5);
- ExpressionResult objToCompare = new ExpressionResult(DataType.DECIMAL, 6);
- boolean result = expressionResult.equals(objToCompare);
- assertFalse(result);
- }
-
- @Test public void testEqualsForDouble() {
- ExpressionResult expressionResult = new ExpressionResult(DataType.DOUBLE, 5.89);
- ExpressionResult objToCompare = new ExpressionResult(DataType.DOUBLE, 6.90);
- boolean result = expressionResult.equals(objToCompare);
- assertFalse(result);
- }
-
- @Test public void testEqualsForException() {
- ExpressionResult expressionResult =
- new ExpressionResult(DataType.TIMESTAMP, "2016-11-07 10:15:09");
- ExpressionResult objToCompare = new ExpressionResult(DataType.DOUBLE, "2016-11-06 10:15:09");
- boolean result = expressionResult.equals(objToCompare);
- assertFalse(result);
- }
-
- @Test public void testEqualsForLongAndTimeStamp() {
- ExpressionResult expressionResult =
- new ExpressionResult(DataType.TIMESTAMP, new Long(11111111111111111L));
- ExpressionResult objToCompare =
- new ExpressionResult(DataType.LONG, new Long(11111111111111111L));
- boolean result = expressionResult.equals(objToCompare);
- assertTrue(result);
- }
-
- @Test public void compareToForInt() {
- ExpressionResult obj = new ExpressionResult(DataType.INT, 5);
- ExpressionResult expressionResult = new ExpressionResult(DataType.INT, 6);
- int actualValue = expressionResult.compareTo(obj);
- int expectedValue = 1;
- assertEquals(expectedValue, actualValue);
-
- }
-
- @Test public void compareToForDecimal() {
- ExpressionResult obj = new ExpressionResult(DataType.DECIMAL, 5);
- ExpressionResult expressionResult = new ExpressionResult(DataType.DECIMAL, 6);
- int actualValue = expressionResult.compareTo(obj);
- int expectedValue = 1;
- assertEquals(expectedValue, actualValue);
- }
-
- @Test public void compareToForException() {
- ExpressionResult obj = new ExpressionResult(DataType.INT, 5);
- ExpressionResult expressionResult = new ExpressionResult(DataType.DECIMAL, 6);
- int actualValue = expressionResult.compareTo(obj);
- int expectedValue = -1;
- assertEquals(expectedValue, actualValue);
- }
-
- @Test public void compareToForTimestamp() {
- ExpressionResult obj = new ExpressionResult(DataType.TIMESTAMP, "2016-11-07 10:15:09");
- ExpressionResult expressionResult =
- new ExpressionResult(DataType.TIMESTAMP, "2016-11-07 10:15:09");
- int actualValue = expressionResult.compareTo(obj);
- int expectedValue = 0;
- assertEquals(expectedValue, actualValue);
- }
-
- @Test public void testGetListAsString() throws FilterIllegalMemberException {
- ExpressionResult obj = new ExpressionResult(DataType.INT, 6);
- List<String> actualValue = obj.getListAsString();
- List<String> expectedValue = new ArrayList<>();
- expectedValue.add("6");
- assertThat(actualValue, is(equalTo(expectedValue)));
- }
-
- @Test public void compareToForString() {
- ExpressionResult obj = new ExpressionResult(DataType.STRING, "2016");
- ExpressionResult expressionResult = new ExpressionResult(DataType.STRING, "2016");
- int actualValue = expressionResult.compareTo(obj);
- int expectedValue = 0;
- assertEquals(expectedValue, actualValue);
- }
-
- @Test public void testIsNullForNull() {
- ExpressionResult obj = new ExpressionResult(DataType.STRING, null);
- boolean actualValue = obj.isNull();
- assertTrue(actualValue);
- }
-
- @Test public void testIsNullForNotNull() {
- ExpressionResult obj = new ExpressionResult(DataType.STRING, "test");
- boolean actualValue = obj.isNull();
- assertFalse(actualValue);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scan/expression/LiteralExpressionTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/expression/LiteralExpressionTest.java b/core/src/test/java/org/apache/carbondata/scan/expression/LiteralExpressionTest.java
deleted file mode 100644
index 1089074..0000000
--- a/core/src/test/java/org/apache/carbondata/scan/expression/LiteralExpressionTest.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.scan.filter.intf.RowImpl;
-
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-public class LiteralExpressionTest {
-
- private LiteralExpression literalExpression;
-
- @Before public void setUp() {
- String value = "testing";
- literalExpression = new LiteralExpression(value, DataType.STRING);
- }
-
- @Test public void testEvaluate() {
- RowImpl rowImpl = new RowImpl();
- rowImpl.setValues(new String[] { "testing" });
- ExpressionResult expectedResult = new ExpressionResult(DataType.STRING, "testing");
- assertEquals(expectedResult, literalExpression.evaluate(rowImpl));
- }
-
- @Test public void testGetExpressionResult() {
- RowImpl rowImpl = new RowImpl();
- rowImpl.setValues(new String[] { "testing" });
- literalExpression.evaluate(rowImpl);
- ExpressionResult expectedResult = new ExpressionResult(DataType.STRING, "testing");
- assertEquals(expectedResult, literalExpression.evaluate(rowImpl));
- }
-
- @Test public void testGetString() {
- String actualValue = literalExpression.getString();
- String expectedValue = "LiteralExpression(testing)";
- assertEquals(expectedValue, actualValue);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scan/expression/conditional/EqualToExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/expression/conditional/EqualToExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/scan/expression/conditional/EqualToExpressionUnitTest.java
deleted file mode 100644
index ce24ad8..0000000
--- a/core/src/test/java/org/apache/carbondata/scan/expression/conditional/EqualToExpressionUnitTest.java
+++ /dev/null
@@ -1,315 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.expression.conditional;
-
-import java.math.BigDecimal;
-import java.sql.Timestamp;
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.intf.RowImpl;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.apache.spark.sql.types.Decimal;
-import org.junit.Test;
-
-import static junit.framework.Assert.assertEquals;
-import static junit.framework.Assert.assertTrue;
-import static org.junit.Assert.assertFalse;
-
-public class EqualToExpressionUnitTest {
-
- static EqualToExpression equalToExpression;
-
- @Test public void testForEqualToExpressionWithGetString() throws Exception {
- ColumnExpression right = new ColumnExpression("name", DataType.STRING);
- right.setColIndex(0);
- equalToExpression = new EqualToExpression(right, right);
- String expected_result = "EqualTo(ColumnExpression(name),ColumnExpression(name))";
- String result = equalToExpression.getString();
- assertEquals(expected_result, result);
- }
-
- @Test public void testEvaluateForEqualToExpressionWithShortDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
- right.setColIndex(0);
- equalToExpression = new EqualToExpression(right, right);
- RowImpl value = new RowImpl();
- Short[] row = { 15 };
- Object objectRow[] = { row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public Short getShort() {
- return 15;
- }
- };
-
- ExpressionResult result = equalToExpression.evaluate(value);
- assertTrue(result.getBoolean());
-
- }
-
- @Test public void testEvaluateForEqualToExpressionWithStringDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("name", DataType.STRING);
- right.setColIndex(0);
- equalToExpression = new EqualToExpression(right, right);
- RowImpl value = new RowImpl();
- String[] row = { "String1" };
- Object objectRow[] = { row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public String getString() {
- return "String1";
- }
- };
-
- ExpressionResult result = equalToExpression.evaluate(value);
- assertTrue(result.getBoolean());
-
- }
-
- @Test public void testEvaluateForEqualToExpressionWithIntDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("number", DataType.INT);
- right.setColIndex(0);
- equalToExpression = new EqualToExpression(right, right);
- RowImpl value = new RowImpl();
- Integer[] row = { 14 };
- Object objectRow[] = { row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public Integer getInt() {
- return 14;
- }
- };
-
- ExpressionResult result = equalToExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForEqualToExpressionWithDoubleDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("contact", DataType.DOUBLE);
- right.setColIndex(0);
- equalToExpression = new EqualToExpression(right, right);
- RowImpl value = new RowImpl();
- Double[] row = { 44D };
- Object objectRow[] = { row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public Double getDouble() {
- return 44D;
- }
- };
-
- ExpressionResult result = equalToExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForEqualToExpressionWithLongDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("contact", DataType.LONG);
- right.setColIndex(0);
- equalToExpression = new EqualToExpression(right, right);
- RowImpl value = new RowImpl();
- Long[] row = { 1234567654321L };
- Object objectRow[] = { row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public Long getLong() {
- return 1234567654321L;
- }
- };
-
- ExpressionResult result = equalToExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForEqualToExpressionWithTimestampDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- try {
- ColumnExpression right = new ColumnExpression("timestamp", DataType.TIMESTAMP);
- right.setColIndex(0);
- equalToExpression = new EqualToExpression(right, right);
- RowImpl value = new RowImpl();
- DateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy");
- Date date = dateFormat.parse("23/09/2007");
- long time = date.getTime();
- Timestamp[] row = { new Timestamp(time) };
- Object objectRow[] = { row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public Long getTime() {
- return 18465213000000L;
- }
- };
-
- ExpressionResult result = equalToExpression.evaluate(value);
- assertTrue(result.getBoolean());
- } catch (ParseException e) {
- System.out.println("Error while parsing " + e.getMessage());
- }
- }
-
- @Test(expected = FilterUnsupportedException.class) public void testForEqualToExpressionForDefaultCase()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("contact", DataType.BOOLEAN);
- right.setColIndex(0);
- equalToExpression = new EqualToExpression(right, right);
- RowImpl value = new RowImpl();
- Boolean[] row = { true };
- Object objectRow[] = { row };
- value.setValues(objectRow);
- ExpressionResult result = equalToExpression.evaluate(value);
- }
-
- @Test public void testEvaluateForEqualToExpressionWithBooleanParameter()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
- right.setColIndex(0);
- equalToExpression = new EqualToExpression(right, right, true);
- RowImpl value = new RowImpl();
- Short[] row = { 15 };
- Object objectRow[] = { row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public Short getShort() {
- return 15;
- }
- };
-
- ExpressionResult result = equalToExpression.evaluate(value);
- assertTrue(result.getBoolean());
-
- }
-
- @Test public void testEvaluateForEqualToExpressionWithLeftAndRightDifferentDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression left = new ColumnExpression("name", DataType.STRING);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("number", DataType.INT);
- right.setColIndex(0);
- equalToExpression = new EqualToExpression(left, right);
- RowImpl value = new RowImpl();
- String[] row1 = { "String1" };
- Integer[] row = { 14 };
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public Integer getInt() {
- return 14;
- }
- };
-
- ExpressionResult result = equalToExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForEqualToExpressionWithIsNullReturnFalse()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
- right.setColIndex(0);
- equalToExpression = new EqualToExpression(right, right);
- RowImpl value = new RowImpl();
- Short[] row = { 15 };
- Object objectRow[] = { row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public boolean isNull() {
- return true;
- }
- };
-
- new MockUp<ExpressionResult>() {
- @Mock public Short getShort() {
- return 15;
- }
- };
-
- ExpressionResult result = equalToExpression.evaluate(value);
- assertFalse(result.getBoolean());
-
- }
-
- @Test public void testEvaluateForEqualToExpressionWithNullWhileCreatingObject()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
- right.setColIndex(0);
- equalToExpression = new EqualToExpression(right, right, true);
- RowImpl value = new RowImpl();
- Short[] row = { 15 };
- Object objectRow[] = { row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public boolean isNull() {
- return true;
- }
- };
-
- new MockUp<ExpressionResult>() {
- @Mock public Short getShort() {
- return 15;
- }
- };
-
- ExpressionResult result = equalToExpression.evaluate(value);
- assertEquals(DataType.BOOLEAN, result.getDataType());
-
- }
-
- @Test public void testEvaluateForEqualToExpressionWithDecimalDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("contact", DataType.DECIMAL);
- right.setColIndex(0);
- equalToExpression = new EqualToExpression(right, right);
- RowImpl value = new RowImpl();
- Decimal[] row = new Decimal[] { Decimal.apply(12345.0) };
- Object objectRow[] = { row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public BigDecimal getDecimal() {
- return new BigDecimal(12345.0);
- }
- };
-
- ExpressionResult result = equalToExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java
deleted file mode 100644
index 4eab74a..0000000
--- a/core/src/test/java/org/apache/carbondata/scan/expression/conditional/GreaterThanEqualToExpressionUnitTest.java
+++ /dev/null
@@ -1,293 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.expression.conditional;
-
-import java.math.BigDecimal;
-import java.sql.Timestamp;
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.intf.RowImpl;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.apache.spark.sql.types.Decimal;
-import org.junit.Test;
-
-import static junit.framework.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-public class GreaterThanEqualToExpressionUnitTest {
-
- static GreaterThanEqualToExpression greaterThanEqualToExpression;
-
- @Test public void testEvaluateForGreaterThanEqualToExpressionWithBothStringISSame()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
- right.setColIndex(1);
- greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
- RowImpl value = new RowImpl();
- String[] row = { "string1" };
- String[] row1 = { "string1" };
- Object objectRow[] = { row, row1 };
-
- new MockUp<ExpressionResult>() {
- @Mock public String getString() {
- return "string1";
- }
- };
- value.setValues(objectRow);
- ExpressionResult result = greaterThanEqualToExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForGreaterThanEqualToExpressionWithShortDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
- right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("id", DataType.SHORT);
- left.setColIndex(1);
- greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
- RowImpl value = new RowImpl();
- Short[] row = { 15 };
- Short[] row1 = { 16 };
- Object objectRow[] = { row1, row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public Short getShort() {
- return 16;
- }
- };
-
- ExpressionResult result = greaterThanEqualToExpression.evaluate(value);
- assertTrue(result.getBoolean());
-
- }
-
- @Test public void testEvaluateForGreaterThanEqualToExpressionWithIntDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("right_number", DataType.INT);
- right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("left_number", DataType.INT);
- left.setColIndex(1);
- greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
- RowImpl value = new RowImpl();
- Integer[] row = { 140 };
- Integer[] row1 = { 145 };
- Object objectRow[] = { row1, row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public Integer getInt() {
- return 145;
- }
- };
-
- ExpressionResult result = greaterThanEqualToExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForGreaterThanEqualToExpressionWithDoubleDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("right_contact", DataType.DOUBLE);
- right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("left_contact", DataType.DOUBLE);
- left.setColIndex(1);
- greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
- RowImpl value = new RowImpl();
- Double[] row = { 44D };
- Double[] row1 = { 45D };
- Object objectRow[] = { row1, row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public Double getDouble() {
- return 45D;
- }
- };
-
- ExpressionResult result = greaterThanEqualToExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForGreaterThanEqualToExpressionWithLongDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("contact", DataType.LONG);
- right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("contact", DataType.LONG);
- left.setColIndex(1);
- greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
- RowImpl value = new RowImpl();
- Long[] row = { 1234567654321L };
- Long[] row1 = { 1234567654321L };
- Object objectRow[] = { row1, row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public Long getLong() {
- return 1234567654321L;
- }
- };
-
- ExpressionResult result = greaterThanEqualToExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForGreaterThanEqualToExpressionWithTimestampDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- try {
- ColumnExpression left = new ColumnExpression("timestamp", DataType.TIMESTAMP);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("timestamp", DataType.TIMESTAMP);
- right.setColIndex(1);
- greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
- RowImpl value = new RowImpl();
- DateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy");
- Date date = dateFormat.parse("23/09/2007");
- long time = date.getTime();
- Timestamp[] row = { new Timestamp(time) };
- Timestamp[] row1 = { new Timestamp(time) };
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public Long getTime() {
- return 18465213000000L;
- }
- };
-
- ExpressionResult result = greaterThanEqualToExpression.evaluate(value);
- assertTrue(result.getBoolean());
- } catch (ParseException e) {
- System.out.println("Error while parsing " + e.getMessage());
- }
- }
-
- @Test(expected = FilterUnsupportedException.class) public void testForGreaterThanEqualToExpressionWithDefaultCase()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("contact", DataType.BOOLEAN);
- right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("contact", DataType.BOOLEAN);
- left.setColIndex(1);
- greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
- RowImpl value = new RowImpl();
- Boolean[] row = { true };
- Object objectRow[] = { row, row };
- value.setValues(objectRow);
- greaterThanEqualToExpression.evaluate(value);
- }
-
- @Test public void testEvaluateForGreaterThanEqualToExpressionWithDecimalDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("contact", DataType.DECIMAL);
- right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("contact", DataType.DECIMAL);
- left.setColIndex(1);
- greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
- RowImpl value = new RowImpl();
- Decimal[] row = new Decimal[] { Decimal.apply(12345.0) };
- Object objectRow[] = { row, row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public BigDecimal getDecimal() {
- return new BigDecimal(12345.0);
- }
- };
-
- ExpressionResult result = greaterThanEqualToExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForGreaterThanEqualToExpressionWithIsNullReturnTrue()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression left = new ColumnExpression("id", DataType.SHORT);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
- right.setColIndex(1);
- greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
- RowImpl value = new RowImpl();
- Short[] row = { 15 };
- Object objectRow[] = { row, row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public boolean isNull() {
- return true;
- }
- };
-
- new MockUp<ExpressionResult>() {
- @Mock public Short getShort() {
- return 15;
- }
- };
-
- ExpressionResult result = greaterThanEqualToExpression.evaluate(value);
- assertFalse(result.getBoolean());
-
- }
-
- @Test public void testEvaluateForGreaterThanEqualToExpressionWithLeftAndRightDifferentDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression left = new ColumnExpression("name", DataType.STRING);
- left.setColIndex(1);
- ColumnExpression right = new ColumnExpression("number", DataType.INT);
- right.setColIndex(0);
- greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
- RowImpl value = new RowImpl();
- String[] row1 = { "String1" };
- Integer[] row = { 14 };
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public Integer getInt() {
- return 14;
- }
- };
-
- ExpressionResult result = greaterThanEqualToExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testForGreaterThanEqualToExpressionWithGetString() throws Exception {
- ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
- right.setColIndex(1);
- greaterThanEqualToExpression = new GreaterThanEqualToExpression(left, right);
- String expected_result = "GreaterThanEqualTo(ColumnExpression(left_name),ColumnExpression(right_name))";
- String result = greaterThanEqualToExpression.getString();
- assertEquals(expected_result, result);
- }
-
-}
-
[06/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
new file mode 100644
index 0000000..1da4415
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
@@ -0,0 +1,392 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.carbondata.core.cache.dictionary.AbstractDictionaryCacheTest;
+import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.datastore.IndexKey;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.metadata.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.LiteralExpression;
+import org.apache.carbondata.core.scan.expression.conditional.ListExpression;
+import org.apache.carbondata.core.scan.filter.intf.RowImpl;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.junit.Before;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertFalse;
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+public class FilterUtilTest extends AbstractDictionaryCacheTest {
+
+ private ColumnSchema columnSchema;
+
+ @Before public void setUp() throws Exception {
+ init();
+ this.databaseName = props.getProperty("database", "testSchema");
+ this.tableName = props.getProperty("tableName", "carbon");
+ this.carbonStorePath = props.getProperty("storePath", "carbonStore");
+ carbonTableIdentifier =
+ new CarbonTableIdentifier(databaseName, tableName, UUID.randomUUID().toString());
+ this.carbonStorePath = props.getProperty("storePath", "carbonStore");
+ columnSchema = new ColumnSchema();
+ columnSchema.setColumnar(true);
+ columnSchema.setColumnName("IMEI");
+ columnSchema.setColumnUniqueId(UUID.randomUUID().toString());
+ columnSchema.setDataType(DataType.STRING);
+ columnSchema.setDimensionColumn(true);
+ }
+
+ @Test public void testCheckIfLeftExpressionRequireEvaluation() {
+ List<Expression> children = new ArrayList<>();
+ ListExpression expression = new ListExpression(children);
+ boolean result = FilterUtil.checkIfLeftExpressionRequireEvaluation(expression);
+ assertTrue(result);
+ }
+
+ @Test
+ public void testCheckIfLeftExpressionRequireEvaluationWithExpressionNotInstanceOfColumnExpression() {
+ ColumnExpression expression = new ColumnExpression("test", DataType.STRING);
+ boolean result = FilterUtil.checkIfLeftExpressionRequireEvaluation(expression);
+ assertFalse(result);
+ }
+
+ @Test public void testNanSafeEqualsDoublesWithUnEqualValues() {
+ Double d1 = new Double(60.67);
+ Double d2 = new Double(60.69);
+ boolean result = FilterUtil.nanSafeEqualsDoubles(d1, d2);
+ assertFalse(result);
+ }
+
+ @Test public void testNanSafeEqualsDoublesWithEqualValues() {
+ Double d1 = new Double(60.67);
+ Double d2 = new Double(60.67);
+ boolean result = FilterUtil.nanSafeEqualsDoubles(d1, d2);
+ assertTrue(result);
+ }
+
+ @Test public void testCompareFilterKeyBasedOnDataTypeForShortValue() {
+ String dictionaryVal = "1";
+ String memberVal = "1";
+ int actualResult =
+ FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataType.SHORT);
+ int expectedResult = 0;
+ assertEquals(expectedResult, actualResult);
+ }
+
+ @Test public void testCompareFilterKeyBasedOnDataTypeForIntValue() {
+ String dictionaryVal = "1000";
+ String memberVal = "1001";
+ int actualResult =
+ FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataType.INT);
+ int expectedResult = -1;
+ assertEquals(expectedResult, actualResult);
+ }
+
+ @Test public void testCompareFilterKeyBasedOnDataTypeForDoubleValue() {
+ String dictionaryVal = "1.90";
+ String memberVal = "1.89";
+ int actualResult =
+ FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataType.DOUBLE);
+ int expectedResult = 1;
+ assertEquals(expectedResult, actualResult);
+ }
+
+ @Test public void testCompareFilterKeyBasedOnDataTypeForLongValue() {
+ String dictionaryVal = "111111111111111";
+ String memberVal = "1111111111111111";
+ int actualResult =
+ FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataType.LONG);
+ int expectedResult = -1;
+ assertEquals(expectedResult, actualResult);
+ }
+
+ @Test public void testCompareFilterKeyBasedOnDataTypeForBooleanValue() {
+ String dictionaryVal = "true";
+ String memberVal = "false";
+ int actualResult =
+ FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataType.BOOLEAN);
+ int expectedResult = 1;
+ assertEquals(expectedResult, actualResult);
+ }
+
+ @Test public void testCompareFilterKeyBasedOnDataTypeForDecimalValue() {
+ String dictionaryVal = "1111111";
+ String memberVal = "1111";
+ int actualResult =
+ FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataType.DECIMAL);
+ int expectedResult = 1;
+ assertEquals(expectedResult, actualResult);
+ }
+
+ @Test public void testCompareFilterKeyBasedOnDataTypeForDefaultValue() {
+ String dictionaryVal = "11.78";
+ String memberVal = "1111.90";
+ int actualResult =
+ FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataType.FLOAT);
+ int expectedResult = -1;
+ assertEquals(expectedResult, actualResult);
+ }
+
+ @Test public void testCompareFilterKeyBasedOnDataTypeForTimestamp() {
+ String dictionaryVal = "2008-01-01 00:00:01";
+ String memberVal = "2008-01-01 00:00:01";
+ int actualValue =
+ FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataType.TIMESTAMP);
+ int expectedValue = 0;
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test public void testCompareFilterKeyBasedOnDataTypeForException() throws Exception {
+ String dictionaryVal = "test";
+ String memberVal = "1";
+ int actualValue =
+ FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataType.INT);
+ int expectedValue = -1;
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test public void testCreateIndexKeyFromResolvedFilterVal() throws Exception {
+ long[] startOrEndKey = new long[] { 0, 10 };
+ byte[] startOrEndKeyForNoDictDimension = { 1, 2 };
+ int[] keys = new int[] { 1, 2 };
+ MultiDimKeyVarLengthGenerator multiDimKeyVarLengthGenerator =
+ new MultiDimKeyVarLengthGenerator(keys);
+ assertTrue(FilterUtil
+ .createIndexKeyFromResolvedFilterVal(startOrEndKey, multiDimKeyVarLengthGenerator,
+ startOrEndKeyForNoDictDimension) != null);
+
+ }
+
+ @Test public void testCheckIfExpressionContainsColumn() {
+ String columnName = "IMEI";
+ Expression expression = new ColumnExpression(columnName, DataType.STRING);
+ boolean result = FilterUtil.checkIfExpressionContainsColumn(expression);
+ assertTrue(result);
+ }
+
+ @Test
+ public void testCheckIfExpressionContainsColumnWithExpressionNotInstanceOfColumnExpression() {
+ String columnName = "IMEI";
+ Expression expression = new LiteralExpression(columnName, DataType.STRING);
+ boolean result = FilterUtil.checkIfExpressionContainsColumn(expression);
+ assertFalse(result);
+ }
+
+ @Test public void testIsExpressionNeedsToResolved() {
+ boolean isIncludeFilter = true;
+ Object obj = "test";
+ LiteralExpression literalExpression = new LiteralExpression(obj, DataType.STRING);
+ boolean result = FilterUtil.isExpressionNeedsToResolved(literalExpression, isIncludeFilter);
+ assertFalse(result);
+ }
+
+ @Test public void testIsExpressionNeedsToResolvedWithDataTypeNullAndIsIncludeFilterFalse() {
+ boolean isIncludeFilter = false;
+ Object obj = "test";
+ LiteralExpression literalExpression = new LiteralExpression(obj, DataType.NULL);
+ boolean result = FilterUtil.isExpressionNeedsToResolved(literalExpression, isIncludeFilter);
+ assertTrue(result);
+ }
+
+ @Test public void testGetMaskKey() {
+ int surrogate = 1;
+ int[] keys = new int[] { 1, 2 };
+ MultiDimKeyVarLengthGenerator multiDimKeyVarLengthGenerator =
+ new MultiDimKeyVarLengthGenerator(keys);
+ int ordinal = 1;
+ int keyOrdinal = 1;
+ int columnGroupOrdinal = 1;
+ int complexTypeOrdinal = 1;
+ ColumnSchema columnSchema = new ColumnSchema();
+ columnSchema.setColumnar(true);
+ columnSchema.setColumnName("IMEI");
+ columnSchema.setColumnUniqueId(UUID.randomUUID().toString());
+ columnSchema.setDataType(DataType.STRING);
+ columnSchema.setDimensionColumn(true);
+ CarbonDimension carbonDimension =
+ new CarbonDimension(columnSchema, ordinal, keyOrdinal, columnGroupOrdinal,
+ complexTypeOrdinal);
+ byte[] expectedResult = new byte[] { 1 };
+ byte[] actualResult =
+ FilterUtil.getMaskKey(surrogate, carbonDimension, multiDimKeyVarLengthGenerator);
+ assertArrayEquals(expectedResult, actualResult);
+ }
+
+ @Test public void testGetFilterListForAllMembersRS() throws Exception {
+ Expression expression = new ColumnExpression("IMEI", DataType.STRING);
+ ColumnExpression columnExpression = new ColumnExpression("IMEI", DataType.STRING);
+ String defaultValues = "test";
+ int defaultSurrogate = 1;
+ boolean isIncludeFilter = true;
+ int ordinal = 1;
+ ColumnSchema dimColumn = new ColumnSchema();
+ dimColumn.setColumnar(true);
+ dimColumn.setColumnName("IMEI");
+ dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
+ dimColumn.setDataType(DataType.STRING);
+ dimColumn.setDimensionColumn(true);
+ final CarbonColumn carbonColumn = new CarbonColumn(dimColumn, ordinal, -1);
+ new MockUp<ColumnExpression>() {
+ @Mock public CarbonColumn getCarbonColumn() {
+ return carbonColumn;
+ }
+ };
+
+ new MockUp<RowImpl>() {
+ @Mock public Object getVal(int index) {
+ return "test";
+ }
+ };
+ assertTrue(FilterUtil
+ .getFilterListForAllMembersRS(expression, columnExpression, defaultValues, defaultSurrogate,
+ isIncludeFilter) instanceof DimColumnFilterInfo);
+ }
+
+ @Test public void testGetFilterListForAllMembersRSWithDefaultValuesEqualsToNull()
+ throws Exception {
+ Expression expression = new ColumnExpression("IMEI", DataType.STRING);
+ ColumnExpression columnExpression = new ColumnExpression("IMEI", DataType.STRING);
+ String defaultValues = CarbonCommonConstants.MEMBER_DEFAULT_VAL;
+ int defaultSurrogate = 1;
+ boolean isIncludeFilter = true;
+ int ordinal = 1;
+ ColumnSchema dimColumn = new ColumnSchema();
+ dimColumn.setColumnar(true);
+ dimColumn.setColumnName("IMEI");
+ dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
+ dimColumn.setDataType(DataType.STRING);
+ dimColumn.setDimensionColumn(true);
+ final CarbonColumn carbonColumn = new CarbonColumn(dimColumn, ordinal, -1);
+ new MockUp<ColumnExpression>() {
+ @Mock public CarbonColumn getCarbonColumn() {
+ return carbonColumn;
+ }
+ };
+
+ new MockUp<RowImpl>() {
+ @Mock public Object getVal(int index) {
+ return "test";
+ }
+ };
+ assertTrue(FilterUtil
+ .getFilterListForAllMembersRS(expression, columnExpression, defaultValues, defaultSurrogate,
+ isIncludeFilter) instanceof DimColumnFilterInfo);
+ }
+
+ @Test public void testgetFilterListForRS() throws Exception {
+ Expression expression = new ColumnExpression("IMEI", DataType.STRING);
+ ColumnExpression columnExpression = new ColumnExpression("IMEI", DataType.STRING);
+ String defaultValues = CarbonCommonConstants.MEMBER_DEFAULT_VAL;
+ int defaultSurrogate = 1;
+ int ordinal = 1;
+ final CarbonColumn carbonColumn = new CarbonColumn(columnSchema, ordinal, -1);
+ new MockUp<ColumnExpression>() {
+ @Mock public CarbonColumn getCarbonColumn() {
+ return carbonColumn;
+ }
+ };
+
+ new MockUp<RowImpl>() {
+ @Mock public Object getVal(int index) {
+ return "test";
+ }
+ };
+ assertTrue(FilterUtil.getFilterListForRS(expression, columnExpression, defaultValues,
+ defaultSurrogate) instanceof DimColumnFilterInfo);
+ }
+
+ @Test public void testCheckIfDataTypeNotTimeStamp() {
+ Expression expression = new ColumnExpression("test", DataType.STRING);
+ boolean result = FilterUtil.checkIfDataTypeNotTimeStamp(expression);
+ assertFalse(result);
+ }
+
+ @Test public void testPrepareDefaultEndIndexKey() throws Exception {
+ List<ColumnSchema> columnsInTable = new ArrayList<>();
+ columnsInTable.add(columnSchema);
+ int[] columnCardinality = new int[] { 1, 2 };
+ new MockUp<ColumnSchema>() {
+ @Mock public List<Encoding> getEncodingList() {
+ List<Encoding> encodingList = new ArrayList<>();
+ encodingList.add(Encoding.DICTIONARY);
+ return encodingList;
+ }
+ };
+ SegmentProperties segmentProperties = new SegmentProperties(columnsInTable, columnCardinality);
+ assertTrue(FilterUtil.prepareDefaultEndIndexKey(segmentProperties) instanceof IndexKey);
+ }
+
+ @Test public void testCheckIfRightExpressionRequireEvaluation() {
+ Expression expression = new ColumnExpression("test", DataType.STRING);
+ boolean result = FilterUtil.checkIfRightExpressionRequireEvaluation(expression);
+ assertTrue(result);
+ }
+
+ @Test
+ public void testCheckIfRightExpressionRequireEvaluationWithExpressionIsInstanceOfLiteralExpression() {
+ Expression expression = new LiteralExpression("test", DataType.STRING);
+ boolean result = FilterUtil.checkIfRightExpressionRequireEvaluation(expression);
+ assertFalse(result);
+ }
+
+ @Test public void testGetNoDictionaryValKeyMemberForFilter() {
+ boolean isIncludeFilter = true;
+ AbsoluteTableIdentifier absoluteTableIdentifier =
+ new AbsoluteTableIdentifier(this.carbonStorePath, carbonTableIdentifier);
+ ColumnExpression expression = new ColumnExpression("test", DataType.STRING);
+ List<String> evaluateResultListFinal = new ArrayList<>();
+ evaluateResultListFinal.add("test1");
+ evaluateResultListFinal.add("test2");
+ assertTrue(FilterUtil.getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, isIncludeFilter) instanceof DimColumnFilterInfo);
+ }
+
+ @Test public void testPrepareDefaultStartIndexKey() throws KeyGenException {
+ List<ColumnSchema> columnsInTable = new ArrayList<>();
+ columnsInTable.add(columnSchema);
+ int[] columnCardinality = new int[] { 1, 2 };
+ new MockUp<ColumnSchema>() {
+ @Mock public List<Encoding> getEncodingList() {
+ List<Encoding> encodingList = new ArrayList<>();
+ encodingList.add(Encoding.DICTIONARY);
+ return encodingList;
+ }
+ };
+ SegmentProperties segmentProperties = new SegmentProperties(columnsInTable, columnCardinality);
+ assertTrue(FilterUtil.prepareDefaultStartIndexKey(segmentProperties) instanceof IndexKey);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/scan/result/BatchResultTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/result/BatchResultTest.java b/core/src/test/java/org/apache/carbondata/core/scan/result/BatchResultTest.java
new file mode 100644
index 0000000..7bb70a0
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/result/BatchResultTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.result;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+public class BatchResultTest {
+ private static BatchResult batchResult;
+ private static List<Object[]> rowsList = new ArrayList(2);
+
+ @BeforeClass public static void setUp() {
+ batchResult = new BatchResult();
+ rowsList.add(0, new Integer[] { 1, 2 });
+ rowsList.add(1, new Integer[] { 3 });
+ }
+
+ @Test public void testNext() throws NoSuchElementException {
+ BatchResult rows = new BatchResult();
+ rows.setRows(rowsList);
+ Object[] result = rows.next();
+ assert (result.equals(rowsList.get(0)));
+ }
+
+ @Test(expected = NoSuchElementException.class) public void testNextWithNoSuchElementException() {
+ BatchResult rows = new BatchResult();
+ List emptyList = new ArrayList(2);
+ rows.setRows(emptyList);
+ rows.next();
+ }
+
+ @Test public void testGetRows() {
+ new MockUp<BatchResult>() {
+ @Mock public void $init() {
+ //to be left blank
+ }
+ };
+ BatchResult batchResult = new BatchResult();
+ List<Object[]> list = batchResult.getRows();
+ assertNull("Number of rows is null", list);
+ }
+
+ @Test public void testHasNext() {
+ List<Object[]> list = new ArrayList<>();
+ list.add(0, new Integer[] { 1, 2 });
+ list.add(1, new Integer[] { 1, 2 });
+ batchResult.setRows(list);
+ boolean result = batchResult.hasNext();
+ assert (result);
+ }
+
+ @Test public void testGetRawRow() {
+ List<Object[]> list = new ArrayList<>();
+ list.add(0, new Integer[] { 1, 2 });
+ batchResult.setRows(list);
+ Object[] actualValue = batchResult.getRawRow(0);
+ assert (list.get(0) == actualValue);
+ }
+
+ @Test public void testGetSize() {
+ List<Object[]> list = new ArrayList<>();
+ list.add(0, new Integer[] { 1, 2 });
+ list.add(1, new Integer[] { 1, 2 });
+ batchResult.setRows(list);
+ int actualValue = batchResult.getSize();
+ int expectedValue = 2;
+ assertEquals(expectedValue, actualValue);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResultTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResultTest.java b/core/src/test/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResultTest.java
new file mode 100644
index 0000000..d77f3e6
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResultTest.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.result.impl;
+
+
+public class FilterQueryScannedResultTest {
+
+// private static FilterQueryScannedResult filterQueryScannedResult;
+//
+// @BeforeClass public static void setUp() {
+// BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
+// blockExecutionInfo.setFixedLengthKeySize(2);
+// blockExecutionInfo.setNoDictionaryBlockIndexes(new int[] { 0, 1 });
+// blockExecutionInfo.setDictionaryColumnBlockIndex(new int[] { 0, 1 });
+// Map<Integer, KeyStructureInfo> columnGourpToKeyInfo = new HashMap<>();
+// columnGourpToKeyInfo.put(1, new KeyStructureInfo());
+// blockExecutionInfo.setColumnGroupToKeyStructureInfo(columnGourpToKeyInfo);
+// Map<Integer, GenericQueryType> genericQueryType = new HashMap<>();
+// genericQueryType.put(1, new ArrayQueryType("Query1", "Parent", 1));
+// blockExecutionInfo.setComplexDimensionInfoMap(genericQueryType);
+// blockExecutionInfo.setComplexColumnParentBlockIndexes(new int[] { 1 });
+// QueryDimension[] queryDimensions = { new QueryDimension("Col1"), new QueryDimension("Col2") };
+// blockExecutionInfo.setQueryDimensions(queryDimensions);
+// filterQueryScannedResult = new FilterQueryScannedResult(blockExecutionInfo);
+// filterQueryScannedResult.setIndexes(new int[] { 1, 2, 3, 4 });
+// DimensionChunkAttributes dimensionChunkAttributes = new DimensionChunkAttributes();
+// dimensionChunkAttributes.setEachRowSize(0);
+// ColumnGroupDimensionDataChunk[] columnGroupDimensionDataChunks =
+// { new ColumnGroupDimensionDataChunk(new byte[] { 1, 2 }, dimensionChunkAttributes),
+// new ColumnGroupDimensionDataChunk(new byte[] { 2, 3 }, dimensionChunkAttributes) };
+// filterQueryScannedResult.setDimensionChunks(columnGroupDimensionDataChunks);
+// MeasureColumnDataChunk measureColumnDataChunk = new MeasureColumnDataChunk();
+// filterQueryScannedResult
+// .setMeasureChunks(new MeasureColumnDataChunk[] { measureColumnDataChunk });
+// }
+//
+// @Test public void testToGetDictionaryKeyArray() {
+// new MockUp<ColumnGroupDimensionDataChunk>() {
+// @Mock @SuppressWarnings("unused") public int fillChunkData(byte[] data, int offset, int rowId,
+// KeyStructureInfo restructuringInfo) {
+// return 1;
+// }
+// };
+// byte[] keyArray = filterQueryScannedResult.getDictionaryKeyArray();
+// byte[] expectedResult = { 0, 0 };
+// assertThat(expectedResult, is(equalTo(keyArray)));
+// }
+//
+// @Test public void testToGetDictionaryKeyIntegerArray() {
+// new MockUp<ColumnGroupDimensionDataChunk>() {
+// @Mock @SuppressWarnings("unused")
+// public int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
+// KeyStructureInfo info) {
+// return 1;
+// }
+// };
+// int[] keyArray = filterQueryScannedResult.getDictionaryKeyIntegerArray();
+// int[] expectedResult = { 0, 0 };
+// assertThat(expectedResult, is(equalTo(keyArray)));
+// }
+//
+// @Test public void testToGetComplexTypeKeyArray() {
+// new MockUp<ByteArrayOutputStream>() {
+// @Mock @SuppressWarnings("unused") public synchronized byte toByteArray()[] {
+// return new byte[] { 1, 2, 3 };
+// }
+// };
+// new MockUp<ArrayQueryType>() {
+// @Mock @SuppressWarnings("unused") public void parseBlocksAndReturnComplexColumnByteArray(
+// DimensionColumnDataChunk[] dimensionColumnDataChunks, int rowNumber,
+// DataOutputStream dataOutputStream) throws IOException {
+// }
+// };
+// filterQueryScannedResult.incrementCounter();
+// byte[][] keyArray = filterQueryScannedResult.getComplexTypeKeyArray();
+// byte[][] expectedResult = { { 1, 2, 3 } };
+// assertThat(expectedResult, is(equalTo(keyArray)));
+// }
+//
+// @Test public void testToGetNoDictionaryKeyArray() {
+// new MockUp<ColumnGroupDimensionDataChunk>() {
+// @Mock @SuppressWarnings("unused") public byte[] getChunkData(int rowId) {
+// return new byte[] { 1, 2, 3 };
+// }
+// };
+// byte[][] dictionaryKeyArray = filterQueryScannedResult.getNoDictionaryKeyArray();
+// byte[][] expectedResult = { { 1, 2, 3 }, { 1, 2, 3 } };
+// assertThat(expectedResult, is(equalTo(dictionaryKeyArray)));
+// }
+//
+// @Test public void testToGetNoDictionaryKeyStringArray() {
+// new MockUp<ColumnGroupDimensionDataChunk>() {
+// @Mock @SuppressWarnings("unused") public byte[] getChunkData(int rowId) {
+// return "1".getBytes();
+// }
+// };
+// filterQueryScannedResult.incrementCounter();
+// String[] dictionaryKeyStringArray = filterQueryScannedResult.getNoDictionaryKeyStringArray();
+// String[] expectedResult = { "1", "1" };
+// assertThat(expectedResult, is(equalTo(dictionaryKeyStringArray)));
+// }
+//
+// @Test public void testToGetCurrenrRowId() {
+// int rowId = filterQueryScannedResult.getCurrenrRowId();
+// int expectedResult = 3;
+// assertThat(expectedResult, is(equalTo(rowId)));
+// }
+//
+// @Test public void testToGetDimensionKey() {
+// new MockUp<ColumnGroupDimensionDataChunk>() {
+// @Mock @SuppressWarnings("unused") public byte[] getChunkData(int rowId) {
+// return "1".getBytes();
+// }
+// };
+// byte[] dictionaryKeyStringArray = filterQueryScannedResult.getDimensionKey(0);
+// byte[] expectedResult = "1".getBytes();
+// assertThat(expectedResult, is(equalTo(dictionaryKeyStringArray)));
+// }
+//
+// @Test public void testToGetIsNullMeasureValue() {
+// new MockUp<MeasureColumnDataChunk>() {
+// @Mock @SuppressWarnings("unused") public PresenceMeta getNullValueIndexHolder() {
+// return new PresenceMeta();
+//
+// }
+// };
+// new MockUp<PresenceMeta>() {
+// @Mock @SuppressWarnings("unused") public BitSet getBitSet() {
+// return new BitSet();
+// }
+// };
+// new MockUp<BitSet>() {
+// @Mock @SuppressWarnings("unused") public boolean get(int bitIndex) {
+// return false;
+// }
+// };
+//
+// boolean nullMeasureValue = filterQueryScannedResult.isNullMeasureValue(0);
+// assertThat(false, is(equalTo(nullMeasureValue)));
+// }
+//
+// @Test public void testToGetLongMeasureValue() {
+// new MockUp<MeasureColumnDataChunk>() {
+// @Mock @SuppressWarnings("unused") public CarbonReadDataHolder getMeasureDataHolder() {
+// return new CarbonReadDataHolder();
+//
+// }
+// };
+// new MockUp<CarbonReadDataHolder>() {
+// @Mock @SuppressWarnings("unused") public long getReadableLongValueByIndex(int index) {
+// return 2L;
+// }
+// };
+// long longMeasureValue = filterQueryScannedResult.getLongMeasureValue(0);
+// long expectedResult = 2L;
+// assertThat(expectedResult, is(equalTo(longMeasureValue)));
+// }
+//
+// @Test public void testToGetDoubleMeasureValue() {
+// new MockUp<MeasureColumnDataChunk>() {
+// @Mock @SuppressWarnings("unused") public CarbonReadDataHolder getMeasureDataHolder() {
+// return new CarbonReadDataHolder();
+//
+// }
+// };
+// new MockUp<CarbonReadDataHolder>() {
+// @Mock @SuppressWarnings("unused") public double getReadableDoubleValueByIndex(int index) {
+// return 2.0;
+// }
+// };
+// double longMeasureValue = filterQueryScannedResult.getDoubleMeasureValue(0);
+// double expectedResult = 2.0;
+// assertThat(expectedResult, is(equalTo(longMeasureValue)));
+// }
+//
+// @Test public void testToGetBigDecimalMeasureValue() {
+// new MockUp<MeasureColumnDataChunk>() {
+// @Mock @SuppressWarnings("unused") public CarbonReadDataHolder getMeasureDataHolder() {
+// return new CarbonReadDataHolder();
+//
+// }
+// };
+// new MockUp<CarbonReadDataHolder>() {
+// @Mock @SuppressWarnings("unused")
+// public BigDecimal getReadableBigDecimalValueByIndex(int index) {
+// return new BigDecimal(2);
+// }
+// };
+// BigDecimal longMeasureValue = filterQueryScannedResult.getBigDecimalMeasureValue(0);
+// BigDecimal expectedResult = new BigDecimal(2);
+// assertThat(expectedResult, is(equalTo(longMeasureValue)));
+// }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResultTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResultTest.java b/core/src/test/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResultTest.java
new file mode 100644
index 0000000..3a33052
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResultTest.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.result.impl;
+
+
+public class NonFilterQueryScannedResultTest {
+
+// private static NonFilterQueryScannedResult filterQueryScannedResult;
+//
+// @BeforeClass public static void setUp() {
+// BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
+// blockExecutionInfo.setFixedLengthKeySize(2);
+// blockExecutionInfo.setNoDictionaryBlockIndexes(new int[] { 0, 1 });
+// blockExecutionInfo.setDictionaryColumnBlockIndex(new int[] { 0, 1 });
+// Map<Integer, KeyStructureInfo> columnGourpToKeyInfo = new HashMap<>();
+// columnGourpToKeyInfo.put(1, new KeyStructureInfo());
+// blockExecutionInfo.setColumnGroupToKeyStructureInfo(columnGourpToKeyInfo);
+// Map<Integer, GenericQueryType> genericQueryType = new HashMap<>();
+// genericQueryType.put(1, new ArrayQueryType("Query1", "Parent", 1));
+// blockExecutionInfo.setComplexDimensionInfoMap(genericQueryType);
+// blockExecutionInfo.setComplexColumnParentBlockIndexes(new int[] { 1 });
+// QueryDimension[] queryDimensions = { new QueryDimension("Col1"), new QueryDimension("Col2") };
+// blockExecutionInfo.setQueryDimensions(queryDimensions);
+// filterQueryScannedResult = new NonFilterQueryScannedResult(blockExecutionInfo);
+// DimensionChunkAttributes dimensionChunkAttributes = new DimensionChunkAttributes();
+// dimensionChunkAttributes.setEachRowSize(0);
+// ColumnGroupDimensionDataChunk[] columnGroupDimensionDataChunks =
+// { new ColumnGroupDimensionDataChunk(new byte[] { 1, 2 }, dimensionChunkAttributes),
+// new ColumnGroupDimensionDataChunk(new byte[] { 2, 3 }, dimensionChunkAttributes) };
+// filterQueryScannedResult.setDimensionChunks(columnGroupDimensionDataChunks);
+// MeasureColumnDataChunk measureColumnDataChunk = new MeasureColumnDataChunk();
+// filterQueryScannedResult
+// .setMeasureChunks(new MeasureColumnDataChunk[] { measureColumnDataChunk });
+// }
+//
+// @Test public void testToGetDictionaryKeyArray() {
+// new MockUp<ColumnGroupDimensionDataChunk>() {
+// @Mock @SuppressWarnings("unused") public int fillChunkData(byte[] data, int offset, int rowId,
+// KeyStructureInfo restructuringInfo) {
+// return 1;
+// }
+// };
+// byte[] keyArray = filterQueryScannedResult.getDictionaryKeyArray();
+// byte[] expectedResult = { 0, 0 };
+// assertThat(expectedResult, is(equalTo(keyArray)));
+// }
+//
+// @Test public void testToGetDictionaryKeyIntegerArray() {
+// new MockUp<ColumnGroupDimensionDataChunk>() {
+// @Mock @SuppressWarnings("unused")
+// public int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
+// KeyStructureInfo info) {
+// return 1;
+// }
+// };
+// int[] keyArray = filterQueryScannedResult.getDictionaryKeyIntegerArray();
+// int[] expectedResult = { 0, 0 };
+// assertThat(expectedResult, is(equalTo(keyArray)));
+// }
+//
+// @Test public void testToGetComplexTypeKeyArray() {
+// new MockUp<ByteArrayOutputStream>() {
+// @Mock @SuppressWarnings("unused") public synchronized byte toByteArray()[] {
+// return new byte[] { 1, 2, 3 };
+// }
+// };
+// new MockUp<ArrayQueryType>() {
+// @Mock @SuppressWarnings("unused") public void parseBlocksAndReturnComplexColumnByteArray(
+// DimensionColumnDataChunk[] dimensionColumnDataChunks, int rowNumber,
+// DataOutputStream dataOutputStream) throws IOException {
+// }
+// };
+// filterQueryScannedResult.incrementCounter();
+// byte[][] keyArray = filterQueryScannedResult.getComplexTypeKeyArray();
+// byte[][] expectedResult = { { 1, 2, 3 } };
+// assertThat(expectedResult, is(equalTo(keyArray)));
+// }
+//
+// @Test public void testToGetNoDictionaryKeyArray() {
+// new MockUp<ColumnGroupDimensionDataChunk>() {
+// @Mock @SuppressWarnings("unused") public byte[] getChunkData(int rowId) {
+// return new byte[] { 1, 2, 3 };
+// }
+// };
+// byte[][] dictionaryKeyArray = filterQueryScannedResult.getNoDictionaryKeyArray();
+// byte[][] expectedResult = { { 1, 2, 3 }, { 1, 2, 3 } };
+// assertThat(expectedResult, is(equalTo(dictionaryKeyArray)));
+// }
+//
+// @Test public void testToGetNoDictionaryKeyStringArray() {
+// new MockUp<ColumnGroupDimensionDataChunk>() {
+// @Mock @SuppressWarnings("unused") public byte[] getChunkData(int rowId) {
+// return "1".getBytes();
+// }
+// };
+// filterQueryScannedResult.incrementCounter();
+// String[] dictionaryKeyStringArray = filterQueryScannedResult.getNoDictionaryKeyStringArray();
+// String[] expectedResult = { "1", "1" };
+// assertThat(expectedResult, is(equalTo(dictionaryKeyStringArray)));
+// }
+//
+// @Test public void testToGetCurrenrRowId() {
+// int rowId = filterQueryScannedResult.getCurrenrRowId();
+// int expectedResult = 2;
+// assertThat(expectedResult, is(equalTo(rowId)));
+// }
+//
+// @Test public void testToGetDimensionKey() {
+// new MockUp<ColumnGroupDimensionDataChunk>() {
+// @Mock @SuppressWarnings("unused") public byte[] getChunkData(int rowId) {
+// return "1".getBytes();
+// }
+// };
+// byte[] dictionaryKeyStringArray = filterQueryScannedResult.getDimensionKey(0);
+// byte[] expectedResult = "1".getBytes();
+// assertThat(expectedResult, is(equalTo(dictionaryKeyStringArray)));
+// }
+//
+// @Test public void testToGetIsNullMeasureValue() {
+// new MockUp<MeasureColumnDataChunk>() {
+// @Mock @SuppressWarnings("unused") public PresenceMeta getNullValueIndexHolder() {
+// return new PresenceMeta();
+//
+// }
+// };
+// new MockUp<PresenceMeta>() {
+// @Mock @SuppressWarnings("unused") public BitSet getBitSet() {
+// return new BitSet();
+// }
+// };
+// new MockUp<BitSet>() {
+// @Mock @SuppressWarnings("unused") public boolean get(int bitIndex) {
+// return false;
+// }
+// };
+//
+// boolean nullMeasureValue = filterQueryScannedResult.isNullMeasureValue(0);
+// assertThat(false, is(equalTo(nullMeasureValue)));
+// }
+//
+// @Test public void testToGetLongMeasureValue() {
+// new MockUp<MeasureColumnDataChunk>() {
+// @Mock @SuppressWarnings("unused") public CarbonReadDataHolder getMeasureDataHolder() {
+// return new CarbonReadDataHolder();
+//
+// }
+// };
+// new MockUp<CarbonReadDataHolder>() {
+// @Mock @SuppressWarnings("unused") public long getReadableLongValueByIndex(int index) {
+// return 2L;
+// }
+// };
+// long longMeasureValue = filterQueryScannedResult.getLongMeasureValue(0);
+// long expectedResult = 2L;
+// assertThat(expectedResult, is(equalTo(longMeasureValue)));
+// }
+//
+// @Test public void testToGetDoubleMeasureValue() {
+// new MockUp<MeasureColumnDataChunk>() {
+// @Mock @SuppressWarnings("unused") public CarbonReadDataHolder getMeasureDataHolder() {
+// return new CarbonReadDataHolder();
+//
+// }
+// };
+// new MockUp<CarbonReadDataHolder>() {
+// @Mock @SuppressWarnings("unused") public double getReadableDoubleValueByIndex(int index) {
+// return 2.0;
+// }
+// };
+// double longMeasureValue = filterQueryScannedResult.getDoubleMeasureValue(0);
+// double expectedResult = 2.0;
+// assertThat(expectedResult, is(equalTo(longMeasureValue)));
+// }
+//
+// @Test public void testToGetBigDecimalMeasureValue() {
+// new MockUp<MeasureColumnDataChunk>() {
+// @Mock @SuppressWarnings("unused") public CarbonReadDataHolder getMeasureDataHolder() {
+// return new CarbonReadDataHolder();
+//
+// }
+// };
+// new MockUp<CarbonReadDataHolder>() {
+// @Mock @SuppressWarnings("unused")
+// public BigDecimal getReadableBigDecimalValueByIndex(int index) {
+// return new BigDecimal(2);
+// }
+// };
+// BigDecimal longMeasureValue = filterQueryScannedResult.getBigDecimalMeasureValue(0);
+// BigDecimal expectedResult = new BigDecimal(2);
+// assertThat(expectedResult, is(equalTo(longMeasureValue)));
+// }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapperTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapperTest.java b/core/src/test/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapperTest.java
new file mode 100644
index 0000000..cec14df
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/wrappers/ByteArrayWrapperTest.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.wrappers;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertFalse;
+import static junit.framework.TestCase.assertTrue;
+
+public class ByteArrayWrapperTest {
+ private static ByteArrayWrapper byteArrayWrapper;
+ byte[] dictionaryKey = new byte[] { 1 };
+ byte[][] complexTypesKeys = { { 1 }, { 1 } };
+ byte[][] noDictionaryKeys = new byte[][] { { 1 }, { 1 } };
+
+ @BeforeClass
+
+ public static void setUp() {
+ byteArrayWrapper = new ByteArrayWrapper();
+ }
+
+ @AfterClass public static void tearDown() {
+ byteArrayWrapper = null;
+ }
+
+ @Test public void testHashCodeValue() {
+ byteArrayWrapper.setDictionaryKey(dictionaryKey);
+ byteArrayWrapper.setComplexTypesKeys(complexTypesKeys);
+ byteArrayWrapper.setNoDictionaryKeys(noDictionaryKeys);
+
+ int result = byteArrayWrapper.hashCode();
+ assert (29583456 == result);
+ }
+
+ @Test public void testEqualsWithOtherAsInstanceOfByteArrayWrapper() {
+ ByteArrayWrapper other;
+ other = null;
+ boolean result = byteArrayWrapper.equals(other);
+ assertFalse(result);
+ }
+
+ @Test public void testEqualsWithNoDictionaryKeysOtherNotEqualNoDictionaryKeys() {
+ byte[][] noDictionaryKeysOther = { { 1 } };
+ byteArrayWrapper.setNoDictionaryKeys(noDictionaryKeys);
+ ByteArrayWrapper other = new ByteArrayWrapper();
+ other.setNoDictionaryKeys(noDictionaryKeysOther);
+ boolean result = byteArrayWrapper.equals(other);
+ assertFalse(result);
+ }
+
+ @Test public void testEqualsWithComplexTypesKeysOtherNotEqualComplexTypesKeys() {
+ byte[][] complexTypesKeysOther = { { 1 } };
+ byteArrayWrapper.setComplexTypesKeys(complexTypesKeys);
+ byteArrayWrapper.setNoDictionaryKeys(noDictionaryKeys);
+ ByteArrayWrapper other = new ByteArrayWrapper();
+ other.setComplexTypesKeys(complexTypesKeysOther);
+ other.setNoDictionaryKeys(noDictionaryKeys);
+ boolean result = byteArrayWrapper.equals(other);
+ assertFalse(result);
+ }
+
+ @Test
+
+ public void testEqualsForFirstElementComplexTypesKeysAndOther() {
+ ByteArrayWrapper other = new ByteArrayWrapper();
+ other.setComplexTypesKeys(complexTypesKeys);
+ other.setDictionaryKey(dictionaryKey);
+ other.setNoDictionaryKeys(noDictionaryKeys);
+ byteArrayWrapper.setComplexTypesKeys(complexTypesKeys);
+ byteArrayWrapper.setDictionaryKey(dictionaryKey);
+ byteArrayWrapper.setNoDictionaryKeys(noDictionaryKeys);
+ boolean result = byteArrayWrapper.equals(other);
+ assertTrue(result);
+ }
+
+ @Test public void testCompareTo() {
+ byteArrayWrapper.setDictionaryKey(dictionaryKey);
+ ByteArrayWrapper other = new ByteArrayWrapper();
+ other.setDictionaryKey(dictionaryKey);
+ other.setNoDictionaryKeys(noDictionaryKeys);
+ other.setComplexTypesKeys(complexTypesKeys);
+ byteArrayWrapper.setNoDictionaryKeys(noDictionaryKeys);
+ byteArrayWrapper.setComplexTypesKeys(complexTypesKeys);
+ int actualResult = byteArrayWrapper.compareTo(other);
+ int expectedResult = 0;
+ assertEquals(expectedResult, actualResult);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImplTest.java b/core/src/test/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImplTest.java
new file mode 100644
index 0000000..54f601b
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImplTest.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.stats;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.core.stats.DriverQueryStatisticsRecorderImpl;
+import org.apache.carbondata.core.stats.QueryStatistic;
+import org.apache.carbondata.core.stats.QueryStatisticsConstants;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertNotNull;
+import static org.pentaho.di.core.util.Assert.assertNull;
+
+public class DriverQueryStatisticsRecorderImplTest {
+ private static DriverQueryStatisticsRecorderImpl driverQueryStatisticsRecorderImpl = null;
+ private static QueryStatistic queryStasticsWithSQL_PARSE = null;
+ private static QueryStatistic queryStasticsWithLoadMetaData = null;
+ private static QueryStatistic queryStasticsWithLOAD_BLOCKS_DRIVER = null;
+ private static QueryStatistic queryStasticsWithBLOCK_ALLOCATION = null;
+ private static QueryStatistic queryStasticsWithBLOCK_IDENTIFICATION = null;
+
+ private static List<QueryStatistic> queryStatisticList;
+
+ @BeforeClass public static void setUp() {
+ driverQueryStatisticsRecorderImpl = DriverQueryStatisticsRecorderImpl.getInstance();
+ queryStasticsWithSQL_PARSE = new QueryStatistic();
+ queryStasticsWithLoadMetaData = new QueryStatistic();
+ queryStasticsWithLOAD_BLOCKS_DRIVER = new QueryStatistic();
+ queryStasticsWithBLOCK_ALLOCATION = new QueryStatistic();
+ queryStasticsWithBLOCK_IDENTIFICATION = new QueryStatistic();
+ queryStatisticList = new ArrayList<>();
+
+ }
+
+ @Test public void testCollectDriverStatistics() {
+ queryStasticsWithLOAD_BLOCKS_DRIVER
+ .addCountStatistic(QueryStatisticsConstants.LOAD_BLOCKS_DRIVER, 3);
+ queryStasticsWithLOAD_BLOCKS_DRIVER
+ .addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_DRIVER, 5);
+ queryStasticsWithLOAD_BLOCKS_DRIVER
+ .addFixedTimeStatistic(QueryStatisticsConstants.LOAD_BLOCKS_DRIVER, 5);
+ queryStatisticList.add(queryStasticsWithLOAD_BLOCKS_DRIVER);
+ queryStasticsWithBLOCK_ALLOCATION
+ .addCountStatistic(QueryStatisticsConstants.BLOCK_ALLOCATION, 3);
+ queryStasticsWithBLOCK_ALLOCATION.addStatistics(QueryStatisticsConstants.BLOCK_ALLOCATION, 5);
+ queryStasticsWithLOAD_BLOCKS_DRIVER
+ .addFixedTimeStatistic(QueryStatisticsConstants.LOAD_BLOCKS_DRIVER, 5);
+ queryStatisticList.add(queryStasticsWithBLOCK_ALLOCATION);
+ queryStasticsWithBLOCK_IDENTIFICATION
+ .addCountStatistic(QueryStatisticsConstants.BLOCK_IDENTIFICATION, 3);
+ queryStasticsWithBLOCK_IDENTIFICATION
+ .addStatistics(QueryStatisticsConstants.BLOCK_IDENTIFICATION, 5);
+ queryStatisticList.add(queryStasticsWithBLOCK_IDENTIFICATION);
+ queryStasticsWithLoadMetaData.addCountStatistic(QueryStatisticsConstants.LOAD_META, 3);
+ queryStasticsWithLoadMetaData.addStatistics(QueryStatisticsConstants.LOAD_META, 5);
+ queryStatisticList.add(queryStasticsWithLoadMetaData);
+ queryStasticsWithSQL_PARSE.addCountStatistic(QueryStatisticsConstants.SQL_PARSE, 3);
+ queryStasticsWithSQL_PARSE.addStatistics(QueryStatisticsConstants.SQL_PARSE, 5);
+ queryStatisticList.add(queryStasticsWithSQL_PARSE);
+
+ String result =
+ driverQueryStatisticsRecorderImpl.collectDriverStatistics(queryStatisticList, "query1");
+ assertNotNull(result);
+ }
+
+ @Test public void testCollectDriverStatisticsWithBlock_Allocation() {
+ queryStatisticList.clear();
+ queryStasticsWithBLOCK_ALLOCATION
+ .addCountStatistic(QueryStatisticsConstants.BLOCK_ALLOCATION, 3);
+ queryStasticsWithBLOCK_ALLOCATION.addStatistics(QueryStatisticsConstants.BLOCK_ALLOCATION, 5);
+ queryStatisticList.add(queryStasticsWithBLOCK_ALLOCATION);
+ String result =
+ driverQueryStatisticsRecorderImpl.collectDriverStatistics(queryStatisticList, "query1");
+ assertNull(result);
+ }
+
+ @Test public void testCollectDriverStatisticsWithBlock_AllocationAndBlock_Identification() {
+ queryStatisticList.clear();
+ queryStasticsWithBLOCK_ALLOCATION
+ .addCountStatistic(QueryStatisticsConstants.BLOCK_ALLOCATION, 3);
+ queryStasticsWithBLOCK_ALLOCATION.addStatistics(QueryStatisticsConstants.BLOCK_ALLOCATION, 5);
+ queryStatisticList.add(queryStasticsWithBLOCK_ALLOCATION);
+ queryStasticsWithBLOCK_IDENTIFICATION
+ .addCountStatistic(QueryStatisticsConstants.BLOCK_IDENTIFICATION, 3);
+ queryStasticsWithBLOCK_IDENTIFICATION
+ .addStatistics(QueryStatisticsConstants.BLOCK_IDENTIFICATION, 5);
+ queryStatisticList.add(queryStasticsWithBLOCK_IDENTIFICATION);
+ String result =
+ driverQueryStatisticsRecorderImpl.collectDriverStatistics(queryStatisticList, "query1");
+ assertNotNull(result);
+ }
+
+ @Test
+ public void testCollectDriverStatisticsWithBlock_AllocationAndBlock_IdentificationForException() {
+ queryStatisticList.clear();
+
+ driverQueryStatisticsRecorderImpl = DriverQueryStatisticsRecorderImpl.getInstance();
+
+ String result =
+ driverQueryStatisticsRecorderImpl.collectDriverStatistics(queryStatisticList, "query1");
+ assertNull(result);
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/stats/QueryStasticsRecorderImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/stats/QueryStasticsRecorderImplTest.java b/core/src/test/java/org/apache/carbondata/core/stats/QueryStasticsRecorderImplTest.java
new file mode 100644
index 0000000..ad5fd22
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/stats/QueryStasticsRecorderImplTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.stats;
+
+import org.apache.carbondata.core.stats.QueryStatistic;
+import org.apache.carbondata.core.stats.QueryStatisticsConstants;
+import org.apache.carbondata.core.stats.QueryStatisticsRecorderImpl;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertNotNull;
+
+public class QueryStasticsRecorderImplTest {
+
+ private static QueryStatisticsRecorderImpl queryStasticsRecorderImpl = null;
+ private static QueryStatistic queryStatistic = null;
+ private static QueryStatistic queryStatisticWithLOAD_BLOCKS_EXECUTOR = null;
+ private static QueryStatistic queryStatisticWithSCAN_BLOCKS_TIME = null;
+ private static QueryStatistic queryStatisticWithSCAN_BLOCKS_NUM = null;
+ private static QueryStatistic queryStatisticWithLOAD_DICTIONARY = null;
+ private static QueryStatistic queryStatisticWithRESULT_SIZE = null;
+ private static QueryStatistic queryStatisticWithEXECUTOR_PART = null;
+ private static QueryStatistic queryStatisticWithTOTAL_BLOCKLET_NUM = null;
+ private static QueryStatistic queryStatisticWithVALID_SCAN_BLOCKLET_NUM = null;
+
+ @BeforeClass public static void setUp() {
+ queryStatisticWithLOAD_BLOCKS_EXECUTOR = new QueryStatistic();
+ queryStasticsRecorderImpl = new QueryStatisticsRecorderImpl(System.nanoTime() + "");
+ queryStasticsRecorderImpl.logStatisticsAsTableDriver();
+ queryStatisticWithLOAD_BLOCKS_EXECUTOR
+ .addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_EXECUTOR, 5L);
+ queryStatisticWithLOAD_BLOCKS_EXECUTOR
+ .addCountStatistic(QueryStatisticsConstants.LOAD_BLOCKS_EXECUTOR, 5L);
+ queryStatisticWithLOAD_BLOCKS_EXECUTOR
+ .addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_EXECUTOR, 5L);
+ queryStasticsRecorderImpl.logStatistics();
+ queryStasticsRecorderImpl.recordStatistics(queryStatisticWithLOAD_BLOCKS_EXECUTOR);
+ queryStatisticWithSCAN_BLOCKS_TIME = new QueryStatistic();
+ queryStatisticWithSCAN_BLOCKS_TIME.addStatistics(QueryStatisticsConstants.SCAN_BLOCKS_TIME, 5L);
+ queryStatisticWithSCAN_BLOCKS_TIME
+ .addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKS_TIME, 5L);
+ queryStatisticWithSCAN_BLOCKS_TIME.addStatistics(QueryStatisticsConstants.SCAN_BLOCKS_TIME, 5L);
+ queryStasticsRecorderImpl.logStatistics();
+ queryStasticsRecorderImpl.recordStatistics(queryStatisticWithSCAN_BLOCKS_TIME);
+ queryStatisticWithSCAN_BLOCKS_NUM = new QueryStatistic();
+ queryStatisticWithSCAN_BLOCKS_NUM.addStatistics(QueryStatisticsConstants.SCAN_BLOCKS_NUM, 5L);
+ queryStatisticWithSCAN_BLOCKS_NUM
+ .addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKS_NUM, 5L);
+ queryStatisticWithSCAN_BLOCKS_NUM.addStatistics(QueryStatisticsConstants.SCAN_BLOCKS_NUM, 5L);
+ queryStasticsRecorderImpl.logStatistics();
+ queryStasticsRecorderImpl.recordStatistics(queryStatisticWithSCAN_BLOCKS_NUM);
+ queryStatisticWithLOAD_DICTIONARY = new QueryStatistic();
+ queryStatisticWithLOAD_DICTIONARY.addStatistics(QueryStatisticsConstants.LOAD_DICTIONARY, 5L);
+ queryStatisticWithLOAD_DICTIONARY
+ .addCountStatistic(QueryStatisticsConstants.LOAD_DICTIONARY, 5L);
+ queryStatisticWithLOAD_DICTIONARY.addStatistics(QueryStatisticsConstants.LOAD_DICTIONARY, 5L);
+ queryStasticsRecorderImpl.logStatistics();
+ queryStasticsRecorderImpl.recordStatistics(queryStatisticWithLOAD_DICTIONARY);
+ queryStatisticWithRESULT_SIZE = new QueryStatistic();
+ queryStatisticWithRESULT_SIZE.addStatistics(QueryStatisticsConstants.RESULT_SIZE, 5L);
+ queryStatisticWithRESULT_SIZE.addCountStatistic(QueryStatisticsConstants.RESULT_SIZE, 5L);
+ queryStatisticWithRESULT_SIZE.addStatistics(QueryStatisticsConstants.RESULT_SIZE, 5L);
+ queryStasticsRecorderImpl.logStatistics();
+ queryStasticsRecorderImpl.recordStatistics(queryStatisticWithRESULT_SIZE);
+ queryStatisticWithEXECUTOR_PART = new QueryStatistic();
+ queryStatisticWithEXECUTOR_PART.addStatistics(QueryStatisticsConstants.EXECUTOR_PART, 5L);
+ queryStatisticWithEXECUTOR_PART.addCountStatistic(QueryStatisticsConstants.EXECUTOR_PART, 5L);
+ queryStatisticWithEXECUTOR_PART.addStatistics(QueryStatisticsConstants.EXECUTOR_PART, 5L);
+ queryStasticsRecorderImpl.logStatistics();
+ queryStasticsRecorderImpl.recordStatistics(queryStatisticWithEXECUTOR_PART);
+ queryStatisticWithTOTAL_BLOCKLET_NUM = new QueryStatistic();
+ queryStatisticWithTOTAL_BLOCKLET_NUM
+ .addStatistics(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM, 5L);
+ queryStatisticWithTOTAL_BLOCKLET_NUM
+ .addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM, 5L);
+ queryStatisticWithTOTAL_BLOCKLET_NUM
+ .addStatistics(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM, 5L);
+ queryStasticsRecorderImpl.logStatistics();
+ queryStasticsRecorderImpl.recordStatistics(queryStatisticWithTOTAL_BLOCKLET_NUM);
+ queryStatisticWithVALID_SCAN_BLOCKLET_NUM = new QueryStatistic();
+ queryStatisticWithVALID_SCAN_BLOCKLET_NUM
+ .addStatistics(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM, 5L);
+ queryStatisticWithVALID_SCAN_BLOCKLET_NUM
+ .addCountStatistic(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM, 5L);
+ queryStatisticWithVALID_SCAN_BLOCKLET_NUM
+ .addStatistics(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM, 5L);
+ queryStasticsRecorderImpl.logStatistics();
+ queryStasticsRecorderImpl.recordStatistics(queryStatisticWithVALID_SCAN_BLOCKLET_NUM);
+ queryStasticsRecorderImpl.logStatisticsAsTableExecutor();
+ }
+
+ @Test public void testcollectExecutorStatistics() {
+ assertNotNull(queryStasticsRecorderImpl.collectExecutorStatistics());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
index d8b0f40..5855262 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
@@ -24,7 +24,7 @@ import mockit.MockUp;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.metadata.index.BlockIndexInfo;
-import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
import org.apache.carbondata.core.metadata.BlockletInfoColumnar;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.format.*;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
index 2643758..7f6edd1 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
@@ -31,12 +31,12 @@ import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.datastorage.store.columnar.ColumnGroupModel;
-import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastorage.store.filesystem.LocalCarbonFile;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.columnar.ColumnGroupModel;
+import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastorage.filesystem.LocalCarbonFile;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
-import org.apache.carbondata.scan.model.QueryDimension;
+import org.apache.carbondata.core.scan.model.QueryDimension;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.AfterClass;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java b/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java
index 1a1b5c8..f1ee0a0 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java
@@ -26,9 +26,9 @@ import org.apache.carbondata.core.ColumnarFormatVersion;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.metadata.blocklet.SegmentInfo;
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
-import org.apache.carbondata.core.datastorage.store.impl.FileHolderImpl;
+import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.impl.FileHolderImpl;
import org.apache.carbondata.core.reader.CarbonFooterReader;
import org.apache.carbondata.core.reader.CarbonIndexFileReader;
import org.apache.carbondata.core.reader.ThriftReader;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java
index 6772c9d..17fe306 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java
@@ -23,12 +23,29 @@ import static org.junit.Assert.assertTrue;
import java.nio.ByteBuffer;
-import org.apache.carbondata.core.datastorage.store.compression.MeasureMetaDataModel;
-import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastorage.store.compression.decimal.*;
-import org.apache.carbondata.core.datastorage.store.compression.nondecimal.*;
-import org.apache.carbondata.core.datastorage.store.compression.none.*;
+import org.apache.carbondata.core.datastorage.compression.MeasureMetaDataModel;
+import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastorage.compression.decimal.CompressionMaxMinByte;
+import org.apache.carbondata.core.datastorage.compression.decimal.CompressionMaxMinDefault;
+import org.apache.carbondata.core.datastorage.compression.decimal.CompressionMaxMinInt;
+import org.apache.carbondata.core.datastorage.compression.decimal.CompressionMaxMinLong;
+import org.apache.carbondata.core.datastorage.compression.decimal.CompressionMaxMinShort;
+import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalByte;
+import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalDefault;
+import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalInt;
+import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalLong;
+import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalMaxMinByte;
+import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalMaxMinDefault;
+import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalMaxMinInt;
+import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalMaxMinLong;
+import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalMaxMinShort;
+import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalShort;
+import org.apache.carbondata.core.datastorage.compression.none.CompressionNoneByte;
+import org.apache.carbondata.core.datastorage.compression.none.CompressionNoneDefault;
+import org.apache.carbondata.core.datastorage.compression.none.CompressionNoneInt;
+import org.apache.carbondata.core.datastorage.compression.none.CompressionNoneLong;
+import org.apache.carbondata.core.datastorage.compression.none.CompressionNoneShort;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java b/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java
index ad0abd7..6037ee8 100644
--- a/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java
@@ -39,8 +39,8 @@ import org.apache.carbondata.core.ColumnIdentifier;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk;
import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReaderImpl;
import org.apache.carbondata.core.reader.CarbonDictionaryReaderImpl;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java b/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java
index b8fd0b8..0036265 100644
--- a/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java
@@ -30,9 +30,9 @@ import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.metadata.BlockletInfoColumnar;
import org.apache.carbondata.core.reader.CarbonFooterReader;
import org.apache.carbondata.core.util.CarbonMetadataUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedResultCollectorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedResultCollectorTest.java b/core/src/test/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedResultCollectorTest.java
deleted file mode 100644
index 1ae04c5..0000000
--- a/core/src/test/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedResultCollectorTest.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.collector.impl;
-
-
-public class DictionaryBasedResultCollectorTest {
-
-// private static DictionaryBasedResultCollector dictionaryBasedResultCollector;
-// private static BlockExecutionInfo blockExecutionInfo;
-//
-// @BeforeClass public static void setUp() {
-// blockExecutionInfo = new BlockExecutionInfo();
-// KeyStructureInfo keyStructureInfo = new KeyStructureInfo();
-// blockExecutionInfo.setKeyStructureInfo(keyStructureInfo);
-// AggregatorInfo aggregatorInfo = new AggregatorInfo();
-// aggregatorInfo.setMeasureOrdinals(new int[] { 10, 20, 30, 40 });
-// aggregatorInfo.setMeasureExists(new boolean[] { true, false, false, false });
-// aggregatorInfo.setDefaultValues(new Object[] { 1, 2, 3, 4 });
-// aggregatorInfo.setMeasureDataTypes(
-// new DataType[] { DataType.INT, DataType.TIMESTAMP, DataType.INT, DataType.INT });
-// blockExecutionInfo.setAggregatorInfo(aggregatorInfo);
-// QueryDimension queryDimension1 = new QueryDimension("QDCol1");
-// queryDimension1.setQueryOrder(1);
-// ColumnSchema columnSchema = new ColumnSchema();
-// queryDimension1.setDimension(new CarbonDimension(columnSchema, 0, 0, 0, 0));
-// QueryDimension queryDimension2 = new QueryDimension("QDCol2");
-// queryDimension2.setQueryOrder(2);
-// queryDimension2.setDimension(new CarbonDimension(columnSchema, 1, 1, 1, 1));
-// QueryDimension queryDimension3 = new QueryDimension("QDCol3");
-// queryDimension3.setQueryOrder(3);
-// queryDimension3.setDimension(new CarbonDimension(columnSchema, 2, 0, 0, 0));
-// QueryDimension queryDimension4 = new QueryDimension("QDCol4");
-// queryDimension4.setQueryOrder(4);
-// queryDimension4.setDimension(new CarbonDimension(columnSchema, 3, 0, 0, 0));
-// blockExecutionInfo.setQueryDimensions(
-// new QueryDimension[] { queryDimension1, queryDimension2, queryDimension3,
-// queryDimension4 });
-// QueryMeasure queryMeasure1 = new QueryMeasure("QMCol1");
-// queryMeasure1.setQueryOrder(1);
-// QueryMeasure queryMeasure2 = new QueryMeasure("QMCol2");
-// queryMeasure1.setQueryOrder(2);
-// QueryMeasure queryMeasure3 = new QueryMeasure("QMCol3");
-// queryMeasure1.setQueryOrder(3);
-// QueryMeasure queryMeasure4 = new QueryMeasure("QMCol4");
-// queryMeasure1.setQueryOrder(4);
-// blockExecutionInfo.setQueryMeasures(
-// new QueryMeasure[] { queryMeasure1, queryMeasure2, queryMeasure3, queryMeasure4 });
-// Map<Integer, GenericQueryType> complexDimensionInfoMap = new HashMap<>();
-// complexDimensionInfoMap.put(1, new ArrayQueryType("name1", "parent1", 1));
-// complexDimensionInfoMap.put(2, new ArrayQueryType("name2", "parent2", 2));
-// complexDimensionInfoMap.put(3, new ArrayQueryType("name3", "parent3", 3));
-// complexDimensionInfoMap.put(4, new ArrayQueryType("name4", "parent4", 4));
-// blockExecutionInfo.setComplexDimensionInfoMap(complexDimensionInfoMap);
-// dictionaryBasedResultCollector = new DictionaryBasedResultCollector(blockExecutionInfo);
-// }
-//
-// @Test public void testToCollectData() {
-// new MockUp<CarbonUtil>() {
-// @SuppressWarnings("unused") @Mock boolean[] getDictionaryEncodingArray(
-// QueryDimension[] queryDimensions) {
-// return new boolean[] { true, false, true, true };
-// }
-//
-// @SuppressWarnings("unused") @Mock boolean[] getDirectDictionaryEncodingArray(
-// QueryDimension[] queryDimensions) {
-// return new boolean[] { true, true, false, false };
-// }
-//
-// @SuppressWarnings("unused") @Mock boolean[] getComplexDataTypeArray(
-// QueryDimension[] queryDimensions) {
-// return new boolean[] { false, false, true, false };
-// }
-// };
-// new MockUp<DataTypeUtil>() {
-// @SuppressWarnings("unused") @Mock Object getDataBasedOnDataType(String data,
-// DataType actualDataType) {
-// return 1;
-// }
-// };
-//
-// new MockUp<NonFilterQueryScannedResult>() {
-// @SuppressWarnings("unused") @Mock int[] getDictionaryKeyIntegerArray() {
-// this.getMockInstance().incrementCounter();
-// System.out.println("Mocked");
-// return new int[] { 1, 2 };
-// }
-//
-// @SuppressWarnings("unused") @Mock String[] getNoDictionaryKeyStringArray() {
-// return new String[] { "1", "2" };
-// }
-//
-// @SuppressWarnings("unused") @Mock byte[][] getComplexTypeKeyArray() {
-// return new byte[][] { { 1, 2 }, { 1, 2 } };
-// }
-//
-// @SuppressWarnings("unused") @Mock public MeasureColumnDataChunk getMeasureChunk(int ordinal) {
-// MeasureColumnDataChunk measureColumnDataChunk = new MeasureColumnDataChunk();
-// PresenceMeta presenceMeta = new PresenceMeta();
-// BitSet bitSet = new BitSet();
-// bitSet.set(1);
-// presenceMeta.setBitSet(bitSet);
-// measureColumnDataChunk.setNullValueIndexHolder(presenceMeta);
-// CarbonReadDataHolder carbonReadDataHolder = new CarbonReadDataHolder();
-// carbonReadDataHolder.setReadableLongValues(new long[] { 1 });
-// measureColumnDataChunk.setMeasureDataHolder(carbonReadDataHolder);
-// return measureColumnDataChunk;
-// }
-// };
-//
-// new MockUp<DirectDictionaryKeyGeneratorFactory>() {
-// @SuppressWarnings("unused") @Mock DirectDictionaryGenerator getDirectDictionaryGenerator(
-// DataType dataType) {
-// if (dataType == DataType.TIMESTAMP) return new TimeStampDirectDictionaryGenerator(
-// CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT);
-// else return null;
-// }
-// };
-// new MockUp<TimeStampDirectDictionaryGenerator>() {
-// @SuppressWarnings("unused") @Mock Object getValueFromSurrogate(int key) {
-// return 100L;
-// }
-// };
-//
-// new MockUp<ArrayQueryType>() {
-// @SuppressWarnings("unused") @Mock Object getDataBasedOnDataTypeFromSurrogates(
-// ByteBuffer surrogateData) {
-// return ByteBuffer.wrap("1".getBytes());
-// }
-// };
-//
-// AbstractScannedResult abstractScannedResult =
-// new NonFilterQueryScannedResult(blockExecutionInfo);
-// abstractScannedResult.setNumberOfRows(2);
-// List<Object[]> result = dictionaryBasedResultCollector.collectData(abstractScannedResult, 2);
-// int expectedResult = 2;
-// assertThat(result.size(), is(equalTo(expectedResult)));
-// }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollectorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollectorTest.java b/core/src/test/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollectorTest.java
deleted file mode 100644
index a27da72..0000000
--- a/core/src/test/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollectorTest.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.collector.impl;
-
-import static org.hamcrest.CoreMatchers.is;
-import static org.hamcrest.MatcherAssert.assertThat;
-
-public class RawBasedResultCollectorTest {
-//
-// private static RawBasedResultCollector rawBasedResultCollector;
-// private static BlockExecutionInfo blockExecutionInfo;
-// private static KeyGenerator keyGenerator;
-//
-// @BeforeClass public static void setUp() {
-// keyGenerator = new MockUp<KeyGenerator>() {
-// @SuppressWarnings("unused") @Mock long[] getKeyArray(byte[] key, int[] maskedByteRanges) {
-// return new long[] { 1, 2 };
-// }
-//
-// @SuppressWarnings("unused") @Mock byte[] generateKey(long[] keys) throws KeyGenException {
-// return new byte[] { 1, 2 };
-// }
-//
-// }.getMockInstance();
-//
-// blockExecutionInfo = new BlockExecutionInfo();
-// KeyStructureInfo keyStructureInfo = new KeyStructureInfo();
-// keyStructureInfo.setKeyGenerator(keyGenerator);
-// keyStructureInfo.setMaxKey(new byte[] { 1, 2 });
-// keyStructureInfo.setMaskedBytes(new int[] { 1, 2 });
-// keyStructureInfo.setMaskByteRanges(new int[] { 1, 2 });
-// blockExecutionInfo.setKeyStructureInfo(keyStructureInfo);
-// AggregatorInfo aggregatorInfo = new AggregatorInfo();
-// aggregatorInfo.setMeasureOrdinals(new int[] { 10, 20, 30, 40 });
-// aggregatorInfo.setMeasureExists(new boolean[] { true, false, false, false });
-// aggregatorInfo.setDefaultValues(new Object[] { 1, 2, 3, 4 });
-// aggregatorInfo.setMeasureDataTypes(
-// new DataType[] { DataType.INT, DataType.TIMESTAMP, DataType.INT, DataType.INT });
-// QueryMeasure queryMeasure1 = new QueryMeasure("QMCol1");
-// queryMeasure1.setQueryOrder(1);
-// QueryMeasure queryMeasure2 = new QueryMeasure("QMCol2");
-// queryMeasure1.setQueryOrder(2);
-// QueryMeasure queryMeasure3 = new QueryMeasure("QMCol3");
-// queryMeasure1.setQueryOrder(3);
-// QueryMeasure queryMeasure4 = new QueryMeasure("QMCol4");
-// queryMeasure1.setQueryOrder(4);
-// QueryDimension queryDimension1 = new QueryDimension("QDCol1");
-// queryDimension1.setQueryOrder(1);
-// ColumnSchema columnSchema = new ColumnSchema();
-// queryDimension1.setDimension(new CarbonDimension(columnSchema, 0, 0, 0, 0));
-// QueryDimension queryDimension2 = new QueryDimension("QDCol2");
-// queryDimension2.setQueryOrder(2);
-// queryDimension2.setDimension(new CarbonDimension(columnSchema, 1, 1, 1, 1));
-// QueryDimension queryDimension3 = new QueryDimension("QDCol3");
-// queryDimension3.setQueryOrder(3);
-// queryDimension3.setDimension(new CarbonDimension(columnSchema, 2, 0, 0, 0));
-// QueryDimension queryDimension4 = new QueryDimension("QDCol4");
-// queryDimension4.setQueryOrder(4);
-// queryDimension4.setDimension(new CarbonDimension(columnSchema, 3, 0, 0, 0));
-// blockExecutionInfo.setQueryDimensions(
-// new QueryDimension[] { queryDimension1, queryDimension2, queryDimension3,
-// queryDimension4 });
-// blockExecutionInfo.setQueryMeasures(
-// new QueryMeasure[] { queryMeasure1, queryMeasure2, queryMeasure3, queryMeasure4 });
-// blockExecutionInfo.setFixedKeyUpdateRequired(true);
-// blockExecutionInfo.setAggregatorInfo(aggregatorInfo);
-// blockExecutionInfo.setMaskedByteForBlock(new int[] { 1, 2 });
-// blockExecutionInfo.setBlockKeyGenerator(keyGenerator);
-// rawBasedResultCollector = new RawBasedResultCollector(blockExecutionInfo);
-// }
-//
-// @Test public void testToCollectData() {
-//
-// new MockUp<NonFilterQueryScannedResult>() {
-// @SuppressWarnings("unused") @Mock byte[] getDictionaryKeyArray() {
-// this.getMockInstance().incrementCounter();
-// return new byte[] { 1, 2 };
-// }
-//
-// @SuppressWarnings("unused") @Mock byte[][] getNoDictionaryKeyArray() {
-// return new byte[][] { { 1, 2 } };
-// }
-//
-// @SuppressWarnings("unused") @Mock byte[][] getComplexTypeKeyArray() {
-// return new byte[][] { { 1, 2 }, { 1, 2 } };
-// }
-//
-// @SuppressWarnings("unused") @Mock public MeasureColumnDataChunk getMeasureChunk(int ordinal) {
-// MeasureColumnDataChunk measureColumnDataChunk = new MeasureColumnDataChunk();
-// PresenceMeta presenceMeta = new PresenceMeta();
-// BitSet bitSet = new BitSet();
-// bitSet.set(1);
-// presenceMeta.setBitSet(bitSet);
-// measureColumnDataChunk.setNullValueIndexHolder(presenceMeta);
-// CarbonReadDataHolder carbonReadDataHolder = new CarbonReadDataHolder();
-// carbonReadDataHolder.setReadableLongValues(new long[] { 1 });
-// measureColumnDataChunk.setMeasureDataHolder(carbonReadDataHolder);
-// return measureColumnDataChunk;
-// }
-// };
-//
-// new MockUp<QueryUtil>() {
-// @SuppressWarnings("unused") @Mock byte[] getMaskedKey(byte[] data, byte[] maxKey,
-// int[] maskByteRanges, int byteCount) {
-// return new byte[] { 1, 2 };
-// }
-// };
-//
-// AbstractScannedResult abstractScannedResult =
-// new NonFilterQueryScannedResult(blockExecutionInfo);
-// abstractScannedResult.setNumberOfRows(2);
-// List<Object[]> result = rawBasedResultCollector.collectData(abstractScannedResult, 2);
-// int expectedResult = 2;
-// assertThat(result.size(), is(equalTo(expectedResult)));
-// }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scan/complextypes/ArrayQueryTypeTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/complextypes/ArrayQueryTypeTest.java b/core/src/test/java/org/apache/carbondata/scan/complextypes/ArrayQueryTypeTest.java
deleted file mode 100644
index 8fc5484..0000000
--- a/core/src/test/java/org/apache/carbondata/scan/complextypes/ArrayQueryTypeTest.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.complextypes;
-
-import java.nio.ByteBuffer;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static org.pentaho.di.core.util.Assert.assertNotNull;
-import static org.pentaho.di.core.util.Assert.assertNull;
-
-public class ArrayQueryTypeTest {
-
- private static ArrayQueryType arrayQueryType;
-
- @BeforeClass public static void setUp() {
- String name = "test";
- String parentName = "testParent";
- int blockIndex = 1;
- arrayQueryType = new ArrayQueryType(name, parentName, blockIndex);
- }
-
- @Test public void testGetDataBasedOnDataTypeFromSurrogatesForNull() {
- ByteBuffer surrogateData = ByteBuffer.allocate(10);
- surrogateData.put(0, (byte) 0xFF);
- surrogateData.put(1, (byte) 0xFF);
- surrogateData.put(2, (byte) 0xFF);
- surrogateData.put(3, (byte) 0xFF);
- assertNull(arrayQueryType.getDataBasedOnDataTypeFromSurrogates(surrogateData));
- }
-
- @Test public void testGetDataBasedOnDataTypeFromSurrogates() {
- ByteBuffer surrogateData = ByteBuffer.allocate(10);
- surrogateData.put(3, (byte) 1);
- arrayQueryType.setName("testName");
- arrayQueryType.setParentname("testName");
- arrayQueryType.addChildren(arrayQueryType);
- assertNotNull(arrayQueryType.getDataBasedOnDataTypeFromSurrogates(surrogateData));
- }
-}
[12/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/model/QueryModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/model/QueryModel.java b/core/src/main/java/org/apache/carbondata/scan/model/QueryModel.java
deleted file mode 100644
index 8665463..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/model/QueryModel.java
+++ /dev/null
@@ -1,365 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.model;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.carbondata.core.cache.dictionary.Dictionary;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.datastore.block.TableBlockInfo;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-import org.apache.carbondata.core.stats.QueryStatisticsRecorder;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.update.UpdateVO;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.UnknownExpression;
-import org.apache.carbondata.scan.expression.conditional.ConditionalExpression;
-import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
-
-/**
- * Query model which will have all the detail
- * about the query, This will be sent from driver to executor '
- * This will be refereed to executing the query.
- */
-public class QueryModel implements Serializable {
-
- /**
- * serialization version
- */
- private static final long serialVersionUID = -4674677234007089052L;
- /**
- * this will hold the information about the dictionary dimension
- * which to
- */
- public transient Map<String, Dictionary> columnToDictionaryMapping;
- /**
- * list of dimension selected for in query
- */
- private List<QueryDimension> queryDimension;
- /**
- * list of measure selected in query
- */
- private List<QueryMeasure> queryMeasures;
- /**
- * query id
- */
- private String queryId;
- /**
- * filter tree
- */
- private FilterResolverIntf filterExpressionResolverTree;
-
- /**
- * table block information in which query will be executed
- */
- private List<TableBlockInfo> tableBlockInfos;
- /**
- * absolute table identifier
- */
- private AbsoluteTableIdentifier absoluteTableIdentifier;
- /**
- * To handle most of the computation in query engines like spark and hive, carbon should give
- * raw detailed records to it.
- */
- private boolean forcedDetailRawQuery;
- /**
- * table on which query will be executed
- * TODO need to remove this ad pass only the path
- * and carbon metadata will load the table from metadata file
- */
- private CarbonTable table;
-
- private QueryStatisticsRecorder statisticsRecorder;
-
- private boolean vectorReader;
-
- /**
- * Invalid table blocks, which need to be removed from
- * memory, invalid blocks can be segment which are deleted
- * or compacted
- */
- private List<String> invalidSegmentIds;
- private Map<String, UpdateVO> invalidSegmentBlockIdMap =
- new HashMap<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-
- public QueryModel() {
- tableBlockInfos = new ArrayList<TableBlockInfo>();
- queryDimension = new ArrayList<QueryDimension>();
- queryMeasures = new ArrayList<QueryMeasure>();
- invalidSegmentIds = new ArrayList<>();
- }
-
- public static QueryModel createModel(AbsoluteTableIdentifier absoluteTableIdentifier,
- CarbonQueryPlan queryPlan, CarbonTable carbonTable) {
- QueryModel queryModel = new QueryModel();
- String factTableName = carbonTable.getFactTableName();
- queryModel.setAbsoluteTableIdentifier(absoluteTableIdentifier);
-
- fillQueryModel(queryPlan, carbonTable, queryModel, factTableName);
-
- queryModel.setForcedDetailRawQuery(queryPlan.isRawDetailQuery());
- queryModel.setQueryId(queryPlan.getQueryId());
- return queryModel;
- }
-
- private static void fillQueryModel(CarbonQueryPlan queryPlan, CarbonTable carbonTable,
- QueryModel queryModel, String factTableName) {
- queryModel.setAbsoluteTableIdentifier(carbonTable.getAbsoluteTableIdentifier());
- queryModel.setQueryDimension(queryPlan.getDimensions());
- queryModel.setQueryMeasures(queryPlan.getMeasures());
- if (null != queryPlan.getFilterExpression()) {
- processFilterExpression(queryPlan.getFilterExpression(),
- carbonTable.getDimensionByTableName(factTableName),
- carbonTable.getMeasureByTableName(factTableName));
- }
- //TODO need to remove this code, and executor will load the table
- // from file metadata
- queryModel.setTable(carbonTable);
- }
-
- public static void processFilterExpression(Expression filterExpression,
- List<CarbonDimension> dimensions, List<CarbonMeasure> measures) {
- if (null != filterExpression) {
- if (null != filterExpression.getChildren() && filterExpression.getChildren().size() == 0) {
- if (filterExpression instanceof ConditionalExpression) {
- List<ColumnExpression> listOfCol =
- ((ConditionalExpression) filterExpression).getColumnList();
- for (ColumnExpression expression : listOfCol) {
- setDimAndMsrColumnNode(dimensions, measures, expression);
- }
- }
- }
- for (Expression expression : filterExpression.getChildren()) {
- if (expression instanceof ColumnExpression) {
- setDimAndMsrColumnNode(dimensions, measures, (ColumnExpression) expression);
- } else if (expression instanceof UnknownExpression) {
- UnknownExpression exp = ((UnknownExpression) expression);
- List<ColumnExpression> listOfColExpression = exp.getAllColumnList();
- for (ColumnExpression col : listOfColExpression) {
- setDimAndMsrColumnNode(dimensions, measures, col);
- }
- } else {
- processFilterExpression(expression, dimensions, measures);
- }
- }
- }
-
- }
-
- private static CarbonMeasure getCarbonMetadataMeasure(String name, List<CarbonMeasure> measures) {
- for (CarbonMeasure measure : measures) {
- if (measure.getColName().equalsIgnoreCase(name)) {
- return measure;
- }
- }
- return null;
- }
-
- private static void setDimAndMsrColumnNode(List<CarbonDimension> dimensions,
- List<CarbonMeasure> measures, ColumnExpression col) {
- CarbonDimension dim;
- CarbonMeasure msr;
- String columnName;
- columnName = col.getColumnName();
- dim = CarbonUtil.findDimension(dimensions, columnName);
- col.setCarbonColumn(dim);
- col.setDimension(dim);
- col.setDimension(true);
- if (null == dim) {
- msr = getCarbonMetadataMeasure(columnName, measures);
- col.setCarbonColumn(msr);
- col.setDimension(false);
- }
- }
-
- /**
- * It gets the projection columns
- */
- public CarbonColumn[] getProjectionColumns() {
- CarbonColumn[] carbonColumns =
- new CarbonColumn[getQueryDimension().size() + getQueryMeasures().size()];
- for (QueryDimension dimension : getQueryDimension()) {
- carbonColumns[dimension.getQueryOrder()] = dimension.getDimension();
- }
- for (QueryMeasure msr : getQueryMeasures()) {
- carbonColumns[msr.getQueryOrder()] = msr.getMeasure();
- }
- return carbonColumns;
- }
-
- /**
- * @return the queryDimension
- */
- public List<QueryDimension> getQueryDimension() {
- return queryDimension;
- }
-
- /**
- * @param queryDimension the queryDimension to set
- */
- public void setQueryDimension(List<QueryDimension> queryDimension) {
- this.queryDimension = queryDimension;
- }
-
- /**
- * @return the queryMeasures
- */
- public List<QueryMeasure> getQueryMeasures() {
- return queryMeasures;
- }
-
- /**
- * @param queryMeasures the queryMeasures to set
- */
- public void setQueryMeasures(List<QueryMeasure> queryMeasures) {
- this.queryMeasures = queryMeasures;
- }
-
- /**
- * @return the queryId
- */
- public String getQueryId() {
- return queryId;
- }
-
- /**
- * @param queryId the queryId to set
- */
- public void setQueryId(String queryId) {
- this.queryId = queryId;
- }
-
- /**
- * @return the tableBlockInfos
- */
- public List<TableBlockInfo> getTableBlockInfos() {
- return tableBlockInfos;
- }
-
- /**
- * @param tableBlockInfos the tableBlockInfos to set
- */
- public void setTableBlockInfos(List<TableBlockInfo> tableBlockInfos) {
- this.tableBlockInfos = tableBlockInfos;
- }
-
- /**
- * @return the filterEvaluatorTree
- */
- public FilterResolverIntf getFilterExpressionResolverTree() {
- return filterExpressionResolverTree;
- }
-
- public void setFilterExpressionResolverTree(FilterResolverIntf filterExpressionResolverTree) {
- this.filterExpressionResolverTree = filterExpressionResolverTree;
- }
-
- /**
- * @return the absoluteTableIdentifier
- */
- public AbsoluteTableIdentifier getAbsoluteTableIdentifier() {
- return absoluteTableIdentifier;
- }
-
- /**
- * @param absoluteTableIdentifier the absoluteTableIdentifier to set
- */
- public void setAbsoluteTableIdentifier(AbsoluteTableIdentifier absoluteTableIdentifier) {
- this.absoluteTableIdentifier = absoluteTableIdentifier;
- }
-
- /**
- * @return the table
- */
- public CarbonTable getTable() {
- return table;
- }
-
- /**
- * @param table the table to set
- */
- public void setTable(CarbonTable table) {
- this.table = table;
- }
-
- public boolean isForcedDetailRawQuery() {
- return forcedDetailRawQuery;
- }
-
- public void setForcedDetailRawQuery(boolean forcedDetailRawQuery) {
- this.forcedDetailRawQuery = forcedDetailRawQuery;
- }
-
- /**
- * @return
- */
- public Map<String, Dictionary> getColumnToDictionaryMapping() {
- return columnToDictionaryMapping;
- }
-
- /**
- * @param columnToDictionaryMapping
- */
- public void setColumnToDictionaryMapping(Map<String, Dictionary> columnToDictionaryMapping) {
- this.columnToDictionaryMapping = columnToDictionaryMapping;
- }
-
- public QueryStatisticsRecorder getStatisticsRecorder() {
- return statisticsRecorder;
- }
-
- public void setStatisticsRecorder(QueryStatisticsRecorder statisticsRecorder) {
- this.statisticsRecorder = statisticsRecorder;
- }
-
- public List<String> getInvalidSegmentIds() {
- return invalidSegmentIds;
- }
-
- public void setInvalidSegmentIds(List<String> invalidSegmentIds) {
- this.invalidSegmentIds = invalidSegmentIds;
- }
-
- public boolean isVectorReader() {
- return vectorReader;
- }
-
- public void setVectorReader(boolean vectorReader) {
- this.vectorReader = vectorReader;
- }
- public void setInvalidBlockForSegmentId(List<UpdateVO> invalidSegmentTimestampList) {
- for (UpdateVO anUpdateVO : invalidSegmentTimestampList) {
- this.invalidSegmentBlockIdMap.put(anUpdateVO.getSegmentId(), anUpdateVO);
- }
- }
-
- public Map<String,UpdateVO> getInvalidBlockVOForSegmentId() {
- return invalidSegmentBlockIdMap;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/processor/AbstractDataBlockIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/processor/AbstractDataBlockIterator.java b/core/src/main/java/org/apache/carbondata/scan/processor/AbstractDataBlockIterator.java
deleted file mode 100644
index be5aa4c..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/processor/AbstractDataBlockIterator.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.processor;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.DataRefNode;
-import org.apache.carbondata.core.stats.QueryStatisticsModel;
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.scan.collector.ScannedResultCollector;
-import org.apache.carbondata.scan.collector.impl.DictionaryBasedResultCollector;
-import org.apache.carbondata.scan.collector.impl.DictionaryBasedVectorResultCollector;
-import org.apache.carbondata.scan.collector.impl.RawBasedResultCollector;
-import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.result.AbstractScannedResult;
-import org.apache.carbondata.scan.result.vector.CarbonColumnarBatch;
-import org.apache.carbondata.scan.scanner.BlockletScanner;
-import org.apache.carbondata.scan.scanner.impl.FilterScanner;
-import org.apache.carbondata.scan.scanner.impl.NonFilterScanner;
-
-/**
- * This abstract class provides a skeletal implementation of the
- * Block iterator.
- */
-public abstract class AbstractDataBlockIterator extends CarbonIterator<List<Object[]>> {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(AbstractDataBlockIterator.class.getName());
-
- /**
- * iterator which will be used to iterate over data blocks
- */
- protected CarbonIterator<DataRefNode> dataBlockIterator;
-
- /**
- * result collector which will be used to aggregate the scanned result
- */
- protected ScannedResultCollector scannerResultAggregator;
-
- /**
- * processor which will be used to process the block processing can be
- * filter processing or non filter processing
- */
- protected BlockletScanner blockletScanner;
-
- /**
- * to hold the data block
- */
- protected BlocksChunkHolder blocksChunkHolder;
-
- /**
- * batch size of result
- */
- protected int batchSize;
-
- protected AbstractScannedResult scannedResult;
-
- public AbstractDataBlockIterator(BlockExecutionInfo blockExecutionInfo,
- FileHolder fileReader, int batchSize, QueryStatisticsModel queryStatisticsModel) {
- dataBlockIterator = new BlockletIterator(blockExecutionInfo.getFirstDataBlock(),
- blockExecutionInfo.getNumberOfBlockToScan());
- blocksChunkHolder = new BlocksChunkHolder(blockExecutionInfo.getTotalNumberDimensionBlock(),
- blockExecutionInfo.getTotalNumberOfMeasureBlock());
- blocksChunkHolder.setFileReader(fileReader);
-
- if (blockExecutionInfo.getFilterExecuterTree() != null) {
- blockletScanner = new FilterScanner(blockExecutionInfo, queryStatisticsModel);
- } else {
- blockletScanner = new NonFilterScanner(blockExecutionInfo, queryStatisticsModel);
- }
- if (blockExecutionInfo.isRawRecordDetailQuery()) {
- LOGGER.info("Row based raw collector is used to scan and collect the data");
- this.scannerResultAggregator =
- new RawBasedResultCollector(blockExecutionInfo);
- } else if (blockExecutionInfo.isVectorBatchCollector()) {
- LOGGER.info("Vector based dictionary collector is used to scan and collect the data");
- this.scannerResultAggregator =
- new DictionaryBasedVectorResultCollector(blockExecutionInfo);
- } else {
- LOGGER.info("Row based dictionary collector is used to scan and collect the data");
- this.scannerResultAggregator =
- new DictionaryBasedResultCollector(blockExecutionInfo);
- }
- this.batchSize = batchSize;
- }
-
- public boolean hasNext() {
- if (scannedResult != null && scannedResult.hasNext()) {
- return true;
- } else {
- return dataBlockIterator.hasNext();
- }
- }
-
- protected boolean updateScanner() {
- try {
- if (scannedResult != null && scannedResult.hasNext()) {
- return true;
- } else {
- scannedResult = getNextScannedResult();
- while (scannedResult != null) {
- if (scannedResult.hasNext()) {
- return true;
- }
- scannedResult = getNextScannedResult();
- }
- return false;
- }
- } catch (IOException | FilterUnsupportedException ex) {
- throw new RuntimeException(ex);
- }
- }
-
- private AbstractScannedResult getNextScannedResult()
- throws IOException, FilterUnsupportedException {
- if (dataBlockIterator.hasNext()) {
- blocksChunkHolder.setDataBlock(dataBlockIterator.next());
- blocksChunkHolder.reset();
- return blockletScanner.scanBlocklet(blocksChunkHolder);
- }
- return null;
- }
-
- public abstract void processNextBatch(CarbonColumnarBatch columnarBatch);
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/processor/BlockletIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/processor/BlockletIterator.java b/core/src/main/java/org/apache/carbondata/scan/processor/BlockletIterator.java
deleted file mode 100644
index fa6cac4..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/processor/BlockletIterator.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.processor;
-
-import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.core.datastore.DataRefNode;
-
-/**
- * Below class will be used to iterate over data block
- */
-public class BlockletIterator extends CarbonIterator<DataRefNode> {
- /**
- * data store block
- */
- protected DataRefNode datablock;
- /**
- * block counter to keep a track how many block has been processed
- */
- private int blockCounter;
-
- /**
- * flag to be used to check any more data block is present or not
- */
- private boolean hasNext = true;
-
- /**
- * total number blocks assgned to this iterator
- */
- private long totalNumberOfBlocksToScan;
-
- /**
- * Constructor
- *
- * @param datablock first data block
- * @param totalNumberOfBlocksToScan total number of blocks to be scanned
- */
- public BlockletIterator(DataRefNode datablock, long totalNumberOfBlocksToScan) {
- this.datablock = datablock;
- this.totalNumberOfBlocksToScan = totalNumberOfBlocksToScan;
- }
-
- /**
- * is all the blocks assigned to this iterator has been processed
- */
- @Override public boolean hasNext() {
- return hasNext;
- }
-
- @Override
- /**
- * To get the next block
- * @return next data block
- *
- */
- public DataRefNode next() {
- // get the current blocks
- DataRefNode datablockTemp = datablock;
- // store the next data block
- datablock = datablock.getNextDataRefNode();
- // increment the counter
- blockCounter++;
- // if all the data block is processed then
- // set the has next flag to false
- // or if number of blocks assigned to this iterator is processed
- // then also set the hasnext flag to false
- if (null == datablock || blockCounter >= this.totalNumberOfBlocksToScan) {
- hasNext = false;
- }
- return datablockTemp;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/processor/BlocksChunkHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/processor/BlocksChunkHolder.java b/core/src/main/java/org/apache/carbondata/scan/processor/BlocksChunkHolder.java
deleted file mode 100644
index d3de726..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/processor/BlocksChunkHolder.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.processor;
-
-import org.apache.carbondata.core.datastore.DataRefNode;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-
-/**
- * Block chunk holder which will hold the dimension and
- * measure chunk
- */
-public class BlocksChunkHolder {
-
- /**
- * dimension column data chunk
- */
- private DimensionColumnDataChunk[] dimensionDataChunk;
-
- /**
- * measure column data chunk
- */
- private MeasureColumnDataChunk[] measureDataChunk;
-
- /**
- * file reader which will use to read the block from file
- */
- private FileHolder fileReader;
-
- /**
- * data block
- */
- private DataRefNode dataBlock;
-
- public BlocksChunkHolder(int numberOfDimensionBlock, int numberOfMeasureBlock) {
- dimensionDataChunk = new DimensionColumnDataChunk[numberOfDimensionBlock];
- measureDataChunk = new MeasureColumnDataChunk[numberOfMeasureBlock];
- }
-
- /**
- * @return the dimensionDataChunk
- */
- public DimensionColumnDataChunk[] getDimensionDataChunk() {
- return dimensionDataChunk;
- }
-
- /**
- * @param dimensionDataChunk the dimensionDataChunk to set
- */
- public void setDimensionDataChunk(DimensionColumnDataChunk[] dimensionDataChunk) {
- this.dimensionDataChunk = dimensionDataChunk;
- }
-
- /**
- * @return the measureDataChunk
- */
- public MeasureColumnDataChunk[] getMeasureDataChunk() {
- return measureDataChunk;
- }
-
- /**
- * @param measureDataChunk the measureDataChunk to set
- */
- public void setMeasureDataChunk(MeasureColumnDataChunk[] measureDataChunk) {
- this.measureDataChunk = measureDataChunk;
- }
-
- /**
- * @return the fileReader
- */
- public FileHolder getFileReader() {
- return fileReader;
- }
-
- /**
- * @param fileReader the fileReader to set
- */
- public void setFileReader(FileHolder fileReader) {
- this.fileReader = fileReader;
- }
-
- /**
- * @return the dataBlock
- */
- public DataRefNode getDataBlock() {
- return dataBlock;
- }
-
- /**
- * @param dataBlock the dataBlock to set
- */
- public void setDataBlock(DataRefNode dataBlock) {
- this.dataBlock = dataBlock;
- }
-
- /***
- * To reset the measure chunk and dimension chunk
- * array
- */
- public void reset() {
- for (int i = 0; i < measureDataChunk.length; i++) {
- this.measureDataChunk[i] = null;
- }
- for (int i = 0; i < dimensionDataChunk.length; i++) {
- this.dimensionDataChunk[i] = null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/processor/impl/DataBlockIteratorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/processor/impl/DataBlockIteratorImpl.java b/core/src/main/java/org/apache/carbondata/scan/processor/impl/DataBlockIteratorImpl.java
deleted file mode 100644
index ee68385..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/processor/impl/DataBlockIteratorImpl.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.processor.impl;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.core.stats.QueryStatisticsModel;
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.scan.processor.AbstractDataBlockIterator;
-import org.apache.carbondata.scan.result.vector.CarbonColumnarBatch;
-
-/**
- * Below class will be used to process the block for detail query
- */
-public class DataBlockIteratorImpl extends AbstractDataBlockIterator {
- /**
- * DataBlockIteratorImpl Constructor
- *
- * @param blockExecutionInfo execution information
- */
- public DataBlockIteratorImpl(BlockExecutionInfo blockExecutionInfo, FileHolder fileReader,
- int batchSize, QueryStatisticsModel queryStatisticsModel) {
- super(blockExecutionInfo, fileReader, batchSize, queryStatisticsModel);
- }
-
- /**
- * It scans the block and returns the result with @batchSize
- *
- * @return Result of @batchSize
- */
- public List<Object[]> next() {
- List<Object[]> collectedResult = null;
- if (updateScanner()) {
- collectedResult = this.scannerResultAggregator.collectData(scannedResult, batchSize);
- while (collectedResult.size() < batchSize && updateScanner()) {
- List<Object[]> data = this.scannerResultAggregator
- .collectData(scannedResult, batchSize - collectedResult.size());
- collectedResult.addAll(data);
- }
- } else {
- collectedResult = new ArrayList<>();
- }
- return collectedResult;
- }
-
- public void processNextBatch(CarbonColumnarBatch columnarBatch) {
- if (updateScanner()) {
- this.scannerResultAggregator.collectVectorBatch(scannedResult, columnarBatch);
- while (columnarBatch.getActualSize() < columnarBatch.getBatchSize() && updateScanner()) {
- this.scannerResultAggregator.collectVectorBatch(scannedResult, columnarBatch);
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/result/AbstractScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/result/AbstractScannedResult.java b/core/src/main/java/org/apache/carbondata/scan/result/AbstractScannedResult.java
deleted file mode 100644
index 6c1a166..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/result/AbstractScannedResult.java
+++ /dev/null
@@ -1,570 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.result;
-
-import java.io.ByteArrayOutputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.util.Map;
-
-import org.apache.carbondata.common.iudprocessor.cache.BlockletLevelDeleteDeltaDataCache;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
-import org.apache.carbondata.core.path.CarbonTablePath;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.scan.filter.GenericQueryType;
-import org.apache.carbondata.scan.result.vector.CarbonColumnVector;
-import org.apache.carbondata.scan.result.vector.ColumnVectorInfo;
-
-/**
- * Scanned result class which will store and provide the result on request
- */
-public abstract class AbstractScannedResult {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(AbstractScannedResult.class.getName());
- /**
- * current row number
- */
- protected int currentRow = -1;
- /**
- * row mapping indexes
- */
- protected int[] rowMapping;
- /**
- * key size of the fixed length column
- */
- private int fixedLengthKeySize;
- /**
- * total number of rows
- */
- private int totalNumberOfRows;
- /**
- * to keep track of number of rows process
- */
- protected int rowCounter;
- /**
- * dimension column data chunk
- */
- protected DimensionColumnDataChunk[] dataChunks;
- /**
- * measure column data chunk
- */
- protected MeasureColumnDataChunk[] measureDataChunks;
- /**
- * dictionary column block index in file
- */
- protected int[] dictionaryColumnBlockIndexes;
-
- /**
- * no dictionary column block index in file
- */
- protected int[] noDictionaryColumnBlockIndexes;
-
- /**
- * column group to is key structure info
- * which will be used to get the key from the complete
- * column group key
- * For example if only one dimension of the column group is selected
- * then from complete column group key it will be used to mask the key and
- * get the particular column key
- */
- protected Map<Integer, KeyStructureInfo> columnGroupKeyStructureInfo;
-
- /**
- *
- */
- private Map<Integer, GenericQueryType> complexParentIndexToQueryMap;
-
- private int totalDimensionsSize;
-
- /**
- * blockedId which will be blockId + blocklet number in the block
- */
- private String blockletId;
-
- private long rowId;
-
- /**
- * parent block indexes
- */
- private int[] complexParentBlockIndexes;
-
- protected BlockletLevelDeleteDeltaDataCache blockletDeleteDeltaCache;
-
- public AbstractScannedResult(BlockExecutionInfo blockExecutionInfo) {
- this.fixedLengthKeySize = blockExecutionInfo.getFixedLengthKeySize();
- this.noDictionaryColumnBlockIndexes = blockExecutionInfo.getNoDictionaryBlockIndexes();
- this.dictionaryColumnBlockIndexes = blockExecutionInfo.getDictionaryColumnBlockIndex();
- this.columnGroupKeyStructureInfo = blockExecutionInfo.getColumnGroupToKeyStructureInfo();
- this.complexParentIndexToQueryMap = blockExecutionInfo.getComlexDimensionInfoMap();
- this.complexParentBlockIndexes = blockExecutionInfo.getComplexColumnParentBlockIndexes();
- this.totalDimensionsSize = blockExecutionInfo.getQueryDimensions().length;
- }
-
- /**
- * Below method will be used to set the dimension chunks
- * which will be used to create a row
- *
- * @param dataChunks dimension chunks used in query
- */
- public void setDimensionChunks(DimensionColumnDataChunk[] dataChunks) {
- this.dataChunks = dataChunks;
- }
-
- /**
- * Below method will be used to set the measure column chunks
- *
- * @param measureDataChunks measure data chunks
- */
- public void setMeasureChunks(MeasureColumnDataChunk[] measureDataChunks) {
- this.measureDataChunks = measureDataChunks;
- }
-
- /**
- * Below method will be used to get the chunk based in measure ordinal
- *
- * @param ordinal measure ordinal
- * @return measure column chunk
- */
- public MeasureColumnDataChunk getMeasureChunk(int ordinal) {
- return measureDataChunks[ordinal];
- }
-
- /**
- * Below method will be used to get the key for all the dictionary dimensions
- * which is present in the query
- *
- * @param rowId row id selected after scanning
- * @return return the dictionary key
- */
- protected byte[] getDictionaryKeyArray(int rowId) {
- byte[] completeKey = new byte[fixedLengthKeySize];
- int offset = 0;
- for (int i = 0; i < this.dictionaryColumnBlockIndexes.length; i++) {
- offset += dataChunks[dictionaryColumnBlockIndexes[i]]
- .fillChunkData(completeKey, offset, rowId,
- columnGroupKeyStructureInfo.get(dictionaryColumnBlockIndexes[i]));
- }
- rowCounter++;
- return completeKey;
- }
-
- /**
- * Below method will be used to get the key for all the dictionary dimensions
- * in integer array format which is present in the query
- *
- * @param rowId row id selected after scanning
- * @return return the dictionary key
- */
- protected int[] getDictionaryKeyIntegerArray(int rowId) {
- int[] completeKey = new int[totalDimensionsSize];
- int column = 0;
- for (int i = 0; i < this.dictionaryColumnBlockIndexes.length; i++) {
- column = dataChunks[dictionaryColumnBlockIndexes[i]]
- .fillConvertedChunkData(rowId, column, completeKey,
- columnGroupKeyStructureInfo.get(dictionaryColumnBlockIndexes[i]));
- }
- rowCounter++;
- return completeKey;
- }
-
- /**
- * Fill the column data of dictionary to vector
- */
- public void fillColumnarDictionaryBatch(ColumnVectorInfo[] vectorInfo) {
- int column = 0;
- for (int i = 0; i < this.dictionaryColumnBlockIndexes.length; i++) {
- column = dataChunks[dictionaryColumnBlockIndexes[i]]
- .fillConvertedChunkData(vectorInfo, column,
- columnGroupKeyStructureInfo.get(dictionaryColumnBlockIndexes[i]));
- }
- }
-
- /**
- * Fill the column data to vector
- */
- public void fillColumnarNoDictionaryBatch(ColumnVectorInfo[] vectorInfo) {
- int column = 0;
- for (int i = 0; i < this.noDictionaryColumnBlockIndexes.length; i++) {
- column = dataChunks[noDictionaryColumnBlockIndexes[i]]
- .fillConvertedChunkData(vectorInfo, column,
- columnGroupKeyStructureInfo.get(noDictionaryColumnBlockIndexes[i]));
- }
- }
-
- /**
- * Fill the measure column data to vector
- */
- public void fillColumnarMeasureBatch(ColumnVectorInfo[] vectorInfo, int[] measuresOrdinal) {
- for (int i = 0; i < measuresOrdinal.length; i++) {
- vectorInfo[i].measureVectorFiller
- .fillMeasureVector(measureDataChunks[measuresOrdinal[i]], vectorInfo[i]);
- }
- }
-
- public void fillColumnarComplexBatch(ColumnVectorInfo[] vectorInfos) {
- for (int i = 0; i < vectorInfos.length; i++) {
- int offset = vectorInfos[i].offset;
- int len = offset + vectorInfos[i].size;
- int vectorOffset = vectorInfos[i].vectorOffset;
- CarbonColumnVector vector = vectorInfos[i].vector;
- for (int j = offset; j < len; j++) {
- ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
- DataOutputStream dataOutput = new DataOutputStream(byteStream);
- try {
- vectorInfos[i].genericQueryType.parseBlocksAndReturnComplexColumnByteArray(dataChunks,
- rowMapping == null ? j : rowMapping[j], dataOutput);
- Object data = vectorInfos[i].genericQueryType
- .getDataBasedOnDataTypeFromSurrogates(ByteBuffer.wrap(byteStream.toByteArray()));
- vector.putObject(vectorOffset++, data);
- } catch (IOException e) {
- LOGGER.error(e);
- } finally {
- CarbonUtil.closeStreams(dataOutput);
- CarbonUtil.closeStreams(byteStream);
- }
- }
- }
- }
-
- /**
- * Just increment the counter incase of query only on measures.
- */
- public void incrementCounter() {
- rowCounter++;
- currentRow++;
- }
-
- /**
- * increment the counter.
- */
- public void setRowCounter(int rowCounter) {
- this.rowCounter = rowCounter;
- }
-
- /**
- * Below method will be used to get the dimension data based on dimension
- * ordinal and index
- *
- * @param dimOrdinal dimension ordinal present in the query
- * @param rowId row index
- * @return dimension data based on row id
- */
- protected byte[] getDimensionData(int dimOrdinal, int rowId) {
- return dataChunks[dimOrdinal].getChunkData(rowId);
- }
-
- /**
- * Below method will be used to get the dimension key array
- * for all the no dictionary dimension present in the query
- *
- * @param rowId row number
- * @return no dictionary keys for all no dictionary dimension
- */
- protected byte[][] getNoDictionaryKeyArray(int rowId) {
- byte[][] noDictionaryColumnsKeys = new byte[noDictionaryColumnBlockIndexes.length][];
- int position = 0;
- for (int i = 0; i < this.noDictionaryColumnBlockIndexes.length; i++) {
- noDictionaryColumnsKeys[position++] =
- dataChunks[noDictionaryColumnBlockIndexes[i]].getChunkData(rowId);
- }
- return noDictionaryColumnsKeys;
- }
-
- /**
- * Below method will be used to get the dimension key array
- * for all the no dictionary dimension present in the query
- *
- * @param rowId row number
- * @return no dictionary keys for all no dictionary dimension
- */
- protected String[] getNoDictionaryKeyStringArray(int rowId) {
- String[] noDictionaryColumnsKeys = new String[noDictionaryColumnBlockIndexes.length];
- int position = 0;
- for (int i = 0; i < this.noDictionaryColumnBlockIndexes.length; i++) {
- noDictionaryColumnsKeys[position++] =
- new String(dataChunks[noDictionaryColumnBlockIndexes[i]].getChunkData(rowId));
- }
- return noDictionaryColumnsKeys;
- }
-
- /**
- * @return blockletId
- */
- public String getBlockletId() {
- return blockletId;
- }
-
- /**
- * @param blockletId
- */
- public void setBlockletId(String blockletId) {
- this.blockletId = CarbonTablePath.getShortBlockId(blockletId);
- }
-
- /**
- * @return blockletId
- */
- public long getRowId() {
- return rowId;
- }
-
- /**
- * @param blockletId
- */
- public void setRowId(long rowId) {
- this.rowId = rowId;
- }
-
-
- /**
- * Below method will be used to get the complex type keys array based
- * on row id for all the complex type dimension selected in query
- *
- * @param rowId row number
- * @return complex type key array for all the complex dimension selected in query
- */
- protected byte[][] getComplexTypeKeyArray(int rowId) {
- byte[][] complexTypeData = new byte[complexParentBlockIndexes.length][];
- for (int i = 0; i < complexTypeData.length; i++) {
- GenericQueryType genericQueryType =
- complexParentIndexToQueryMap.get(complexParentBlockIndexes[i]);
- ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
- DataOutputStream dataOutput = new DataOutputStream(byteStream);
- try {
- genericQueryType.parseBlocksAndReturnComplexColumnByteArray(dataChunks, rowId, dataOutput);
- complexTypeData[i] = byteStream.toByteArray();
- } catch (IOException e) {
- LOGGER.error(e);
- } finally {
- CarbonUtil.closeStreams(dataOutput);
- CarbonUtil.closeStreams(byteStream);
- }
- }
- return complexTypeData;
- }
-
- /**
- * @return return the total number of row after scanning
- */
- public int numberOfOutputRows() {
- return this.totalNumberOfRows;
- }
-
- /**
- * to check whether any more row is present in the result
- *
- * @return
- */
- public boolean hasNext() {
- if (rowCounter < this.totalNumberOfRows) {
- return true;
- }
- CarbonUtil.freeMemory(dataChunks, measureDataChunks);
- return false;
- }
-
- /**
- * As this class will be a flyweight object so
- * for one block all the blocklet scanning will use same result object
- * in that case we need to reset the counter to zero so
- * for new result it will give the result from zero
- */
- public void reset() {
- rowCounter = 0;
- currentRow = -1;
- }
-
- /**
- * @param totalNumberOfRows set total of number rows valid after scanning
- */
- public void setNumberOfRows(int totalNumberOfRows) {
- this.totalNumberOfRows = totalNumberOfRows;
- }
-
- /**
- * After applying filter it will return the bit set with the valid row indexes
- * so below method will be used to set the row indexes
- *
- * @param indexes
- */
- public void setIndexes(int[] indexes) {
- this.rowMapping = indexes;
- }
-
- /**
- * Below method will be used to check whether measure value is null or not
- *
- * @param ordinal measure ordinal
- * @param rowIndex row number to be checked
- * @return whether it is null or not
- */
- protected boolean isNullMeasureValue(int ordinal, int rowIndex) {
- return measureDataChunks[ordinal].getNullValueIndexHolder().getBitSet().get(rowIndex);
- }
-
- /**
- * Below method will be used to get the measure value of
- * long type
- *
- * @param ordinal measure ordinal
- * @param rowIndex row number of the measure value
- * @return measure value of long type
- */
- protected long getLongMeasureValue(int ordinal, int rowIndex) {
- return measureDataChunks[ordinal].getMeasureDataHolder().getReadableLongValueByIndex(rowIndex);
- }
-
- /**
- * Below method will be used to get the measure value of double type
- *
- * @param ordinal measure ordinal
- * @param rowIndex row number
- * @return measure value of double type
- */
- protected double getDoubleMeasureValue(int ordinal, int rowIndex) {
- return measureDataChunks[ordinal].getMeasureDataHolder()
- .getReadableDoubleValueByIndex(rowIndex);
- }
-
- /**
- * Below method will be used to get the measure type of big decimal data type
- *
- * @param ordinal ordinal of the of the measure
- * @param rowIndex row number
- * @return measure of big decimal type
- */
- protected BigDecimal getBigDecimalMeasureValue(int ordinal, int rowIndex) {
- return measureDataChunks[ordinal].getMeasureDataHolder()
- .getReadableBigDecimalValueByIndex(rowIndex);
- }
-
- public int getRowCounter() {
- return rowCounter;
- }
-
- /**
- * will return the current valid row id
- *
- * @return valid row id
- */
- public abstract int getCurrenrRowId();
-
- /**
- * @return dictionary key array for all the dictionary dimension
- * selected in query
- */
- public abstract byte[] getDictionaryKeyArray();
-
- /**
- * @return dictionary key array for all the dictionary dimension in integer array forat
- * selected in query
- */
- public abstract int[] getDictionaryKeyIntegerArray();
-
- /**
- * Return the dimension data based on dimension ordinal
- *
- * @param dimensionOrdinal dimension ordinal
- * @return dimension data
- */
- public abstract byte[] getDimensionKey(int dimensionOrdinal);
-
- /**
- * Below method will be used to get the complex type key array
- *
- * @return complex type key array
- */
- public abstract byte[][] getComplexTypeKeyArray();
-
- /**
- * Below method will be used to get the no dictionary key
- * array for all the no dictionary dimension selected in query
- *
- * @return no dictionary key array for all the no dictionary dimension
- */
- public abstract byte[][] getNoDictionaryKeyArray();
-
- /**
- * Below method will be used to get the no dictionary key
- * array in string array format for all the no dictionary dimension selected in query
- *
- * @return no dictionary key array for all the no dictionary dimension
- */
- public abstract String[] getNoDictionaryKeyStringArray();
-
- /**
- * Below method will be used to to check whether measure value
- * is null or for a measure
- *
- * @param ordinal measure ordinal
- * @return is null or not
- */
- public abstract boolean isNullMeasureValue(int ordinal);
-
- /**
- * Below method will be used to get the measure value for measure
- * of long data type
- *
- * @param ordinal measure ordinal
- * @return long value of measure
- */
- public abstract long getLongMeasureValue(int ordinal);
-
- /**
- * Below method will be used to get the value of measure of double
- * type
- *
- * @param ordinal measure ordinal
- * @return measure value
- */
- public abstract double getDoubleMeasureValue(int ordinal);
-
- /**
- * Below method will be used to get the data of big decimal type
- * of a measure
- *
- * @param ordinal measure ordinal
- * @return measure value
- */
- public abstract BigDecimal getBigDecimalMeasureValue(int ordinal);
-
- /**
- *
- * @return BlockletLevelDeleteDeltaDataCache.
- */
- public BlockletLevelDeleteDeltaDataCache getDeleteDeltaDataCache() {
- return blockletDeleteDeltaCache;
- }
-
- /**
- * @param blockletDeleteDeltaCache
- */
- public void setBlockletDeleteDeltaCache(
- BlockletLevelDeleteDeltaDataCache blockletDeleteDeltaCache) {
- this.blockletDeleteDeltaCache = blockletDeleteDeltaCache;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/result/BatchResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/result/BatchResult.java b/core/src/main/java/org/apache/carbondata/scan/result/BatchResult.java
deleted file mode 100644
index 9bb9c21..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/result/BatchResult.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.result;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.NoSuchElementException;
-
-import org.apache.carbondata.common.CarbonIterator;
-
-/**
- * Below class holds the query result
- */
-public class BatchResult extends CarbonIterator<Object[]> {
-
- /**
- * list of keys
- */
- protected List<Object[]> rows;
-
- /**
- * counter to check whether all the records are processed or not
- */
- protected int counter;
-
- public BatchResult() {
- this.rows = new ArrayList<>();
- }
-
- /**
- * Below method will be used to get the rows
- *
- * @return
- */
- public List<Object[]> getRows() {
- return rows;
- }
-
- /**
- * Below method will be used to get the set the values
- *
- * @param rows
- */
- public void setRows(List<Object[]> rows) {
- this.rows = rows;
- }
-
- /**
- * This method will return one row at a time based on the counter given.
- * @param counter
- * @return
- */
- public Object[] getRawRow(int counter) {
- return rows.get(counter);
- }
-
- /**
- * For getting the total size.
- * @return
- */
- public int getSize() {
- return rows.size();
- }
-
-
- /**
- * Returns {@code true} if the iteration has more elements.
- *
- * @return {@code true} if the iteration has more elements
- */
- @Override public boolean hasNext() {
- return counter < rows.size();
- }
-
- /**
- * Returns the next element in the iteration.
- *
- * @return the next element in the iteration
- */
- @Override public Object[] next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
- }
- Object[] row = rows.get(counter);
- counter++;
- return row;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/result/impl/FilterQueryScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/result/impl/FilterQueryScannedResult.java b/core/src/main/java/org/apache/carbondata/scan/result/impl/FilterQueryScannedResult.java
deleted file mode 100644
index e192f95..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/result/impl/FilterQueryScannedResult.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.result.impl;
-
-import java.math.BigDecimal;
-
-import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.scan.result.AbstractScannedResult;
-import org.apache.carbondata.scan.result.vector.ColumnVectorInfo;
-
-/**
- * Result provider class in case of filter query
- * In case of filter query data will be send
- * based on filtered row index
- */
-public class FilterQueryScannedResult extends AbstractScannedResult {
-
- public FilterQueryScannedResult(BlockExecutionInfo tableBlockExecutionInfos) {
- super(tableBlockExecutionInfos);
- }
-
- /**
- * @return dictionary key array for all the dictionary dimension
- * selected in query
- */
- @Override public byte[] getDictionaryKeyArray() {
- ++currentRow;
- return getDictionaryKeyArray(rowMapping[currentRow]);
- }
-
- /**
- * @return dictionary key integer array for all the dictionary dimension
- * selected in query
- */
- @Override public int[] getDictionaryKeyIntegerArray() {
- ++currentRow;
- return getDictionaryKeyIntegerArray(rowMapping[currentRow]);
- }
-
- /**
- * Below method will be used to get the complex type key array
- *
- * @return complex type key array
- */
- @Override public byte[][] getComplexTypeKeyArray() {
- return getComplexTypeKeyArray(rowMapping[currentRow]);
- }
-
- /**
- * Below method will be used to get the no dictionary key
- * array for all the no dictionary dimension selected in query
- *
- * @return no dictionary key array for all the no dictionary dimension
- */
- @Override public byte[][] getNoDictionaryKeyArray() {
- return getNoDictionaryKeyArray(rowMapping[currentRow]);
- }
-
- /**
- * Below method will be used to get the no dictionary key
- * string array for all the no dictionary dimension selected in query
- *
- * @return no dictionary key array for all the no dictionary dimension
- */
- @Override public String[] getNoDictionaryKeyStringArray() {
- return getNoDictionaryKeyStringArray(rowMapping[currentRow]);
- }
-
- /**
- * will return the current valid row id
- *
- * @return valid row id
- */
- @Override public int getCurrenrRowId() {
- return rowMapping[currentRow];
- }
-
- /**
- * Return the dimension data based on dimension ordinal
- *
- * @param dimensionOrdinal dimension ordinal
- * @return dimension data
- */
- @Override public byte[] getDimensionKey(int dimensionOrdinal) {
- return getDimensionData(dimensionOrdinal, rowMapping[currentRow]);
- }
-
- /**
- * Below method will be used to to check whether measure value
- * is null or for a measure
- *
- * @param ordinal measure ordinal
- * @return is null or not
- */
- @Override public boolean isNullMeasureValue(int ordinal) {
- return isNullMeasureValue(ordinal, rowMapping[currentRow]);
- }
-
- /**
- * Below method will be used to get the measure value for measure
- * of long data type
- *
- * @param ordinal measure ordinal
- * @return long value of measure
- */
- @Override public long getLongMeasureValue(int ordinal) {
- return getLongMeasureValue(ordinal, rowMapping[currentRow]);
- }
-
- /**
- * Below method will be used to get the value of measure of double
- * type
- *
- * @param ordinal measure ordinal
- * @return measure value
- */
- @Override public double getDoubleMeasureValue(int ordinal) {
- return getDoubleMeasureValue(ordinal, rowMapping[currentRow]);
- }
-
- /**
- * Below method will be used to get the data of big decimal type
- * of a measure
- *
- * @param ordinal measure ordinal
- * @return measure value
- */
- @Override public BigDecimal getBigDecimalMeasureValue(int ordinal) {
- return getBigDecimalMeasureValue(ordinal, rowMapping[currentRow]);
- }
-
- /**
- * Fill the column data to vector
- */
- public void fillColumnarDictionaryBatch(ColumnVectorInfo[] vectorInfo) {
- int column = 0;
- for (int i = 0; i < this.dictionaryColumnBlockIndexes.length; i++) {
- column = dataChunks[dictionaryColumnBlockIndexes[i]]
- .fillConvertedChunkData(rowMapping, vectorInfo, column,
- columnGroupKeyStructureInfo.get(dictionaryColumnBlockIndexes[i]));
- }
- }
-
- /**
- * Fill the column data to vector
- */
- public void fillColumnarNoDictionaryBatch(ColumnVectorInfo[] vectorInfo) {
- int column = 0;
- for (int i = 0; i < this.noDictionaryColumnBlockIndexes.length; i++) {
- column = dataChunks[noDictionaryColumnBlockIndexes[i]]
- .fillConvertedChunkData(rowMapping, vectorInfo, column,
- columnGroupKeyStructureInfo.get(noDictionaryColumnBlockIndexes[i]));
- }
- }
-
- /**
- * Fill the measure column data to vector
- */
- public void fillColumnarMeasureBatch(ColumnVectorInfo[] vectorInfo, int[] measuresOrdinal) {
- for (int i = 0; i < measuresOrdinal.length; i++) {
- vectorInfo[i].measureVectorFiller
- .fillMeasureVectorForFilter(rowMapping, measureDataChunks[measuresOrdinal[i]],
- vectorInfo[i]);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/result/impl/NonFilterQueryScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/result/impl/NonFilterQueryScannedResult.java b/core/src/main/java/org/apache/carbondata/scan/result/impl/NonFilterQueryScannedResult.java
deleted file mode 100644
index e08d525..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/result/impl/NonFilterQueryScannedResult.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.result.impl;
-
-import java.math.BigDecimal;
-
-import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.scan.result.AbstractScannedResult;
-
-/**
- * Result provide class for non filter query
- * In case of no filter query we need to return
- * complete data
- */
-public class NonFilterQueryScannedResult extends AbstractScannedResult {
-
- public NonFilterQueryScannedResult(BlockExecutionInfo blockExecutionInfo) {
- super(blockExecutionInfo);
- }
-
- /**
- * @return dictionary key array for all the dictionary dimension selected in
- * query
- */
- @Override public byte[] getDictionaryKeyArray() {
- ++currentRow;
- return getDictionaryKeyArray(currentRow);
- }
-
- /**
- * @return dictionary key integer array for all the dictionary dimension
- * selected in query
- */
- @Override public int[] getDictionaryKeyIntegerArray() {
- ++currentRow;
- return getDictionaryKeyIntegerArray(currentRow);
- }
-
- /**
- * Below method will be used to get the complex type key array
- *
- * @return complex type key array
- */
- @Override public byte[][] getComplexTypeKeyArray() {
- return getComplexTypeKeyArray(currentRow);
- }
-
- /**
- * Below method will be used to get the no dictionary key array for all the
- * no dictionary dimension selected in query
- *
- * @return no dictionary key array for all the no dictionary dimension
- */
- @Override public byte[][] getNoDictionaryKeyArray() {
- return getNoDictionaryKeyArray(currentRow);
- }
-
- /**
- * Below method will be used to get the no dictionary key
- * string array for all the no dictionary dimension selected in query
- *
- * @return no dictionary key array for all the no dictionary dimension
- */
- @Override public String[] getNoDictionaryKeyStringArray() {
- return getNoDictionaryKeyStringArray(currentRow);
- }
-
- /**
- * will return the current valid row id
- *
- * @return valid row id
- */
- @Override public int getCurrenrRowId() {
- return currentRow;
- }
-
- /**
- * Return the dimension data based on dimension ordinal
- *
- * @param dimensionOrdinal dimension ordinal
- * @return dimension data
- */
- @Override public byte[] getDimensionKey(int dimensionOrdinal) {
- return getDimensionData(dimensionOrdinal, currentRow);
- }
-
- /**
- * Below method will be used to to check whether measure value is null or
- * for a measure
- *
- * @param ordinal measure ordinal
- * @return is null or not
- */
- @Override public boolean isNullMeasureValue(int ordinal) {
- return isNullMeasureValue(ordinal, currentRow);
- }
-
- /**
- * Below method will be used to get the measure value for measure of long
- * data type
- *
- * @param ordinal measure ordinal
- * @return long value of measure
- */
- @Override public long getLongMeasureValue(int ordinal) {
- return getLongMeasureValue(ordinal, currentRow);
- }
-
- /**
- * Below method will be used to get the value of measure of double type
- *
- * @param ordinal measure ordinal
- * @return measure value
- */
- @Override public double getDoubleMeasureValue(int ordinal) {
- return getDoubleMeasureValue(ordinal, currentRow);
- }
-
- /**
- * Below method will be used to get the data of big decimal type of a
- * measure
- *
- * @param ordinal measure ordinal
- * @return measure value
- */
- @Override public BigDecimal getBigDecimalMeasureValue(int ordinal) {
- return getBigDecimalMeasureValue(ordinal, currentRow);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/result/iterator/AbstractDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/result/iterator/AbstractDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/scan/result/iterator/AbstractDetailQueryResultIterator.java
deleted file mode 100644
index 5d174d3..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/result/iterator/AbstractDetailQueryResultIterator.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.result.iterator;
-
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-
-import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.DataRefNode;
-import org.apache.carbondata.core.datastore.DataRefNodeFinder;
-import org.apache.carbondata.core.datastore.impl.btree.BTreeDataRefNodeFinder;
-import org.apache.carbondata.core.stats.QueryStatistic;
-import org.apache.carbondata.core.stats.QueryStatisticsConstants;
-import org.apache.carbondata.core.stats.QueryStatisticsModel;
-import org.apache.carbondata.core.stats.QueryStatisticsRecorder;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
-import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.scan.model.QueryModel;
-import org.apache.carbondata.scan.processor.AbstractDataBlockIterator;
-import org.apache.carbondata.scan.processor.impl.DataBlockIteratorImpl;
-import org.apache.carbondata.scan.result.vector.CarbonColumnarBatch;
-
-/**
- * In case of detail query we cannot keep all the records in memory so for
- * executing that query are returning a iterator over block and every time next
- * call will come it will execute the block and return the result
- */
-public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterator<E> {
-
- /**
- * LOGGER.
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(AbstractDetailQueryResultIterator.class.getName());
-
- protected ExecutorService execService;
- /**
- * execution info of the block
- */
- protected List<BlockExecutionInfo> blockExecutionInfos;
-
- /**
- * file reader which will be used to execute the query
- */
- protected FileHolder fileReader;
- protected AbstractDataBlockIterator dataBlockIterator;
- protected boolean nextBatch = false;
- /**
- * total time scan the blocks
- */
- protected long totalScanTime;
- /**
- * is the statistic recorded
- */
- protected boolean isStatisticsRecorded;
- /**
- * QueryStatisticsRecorder
- */
- protected QueryStatisticsRecorder recorder;
- /**
- * number of cores which can be used
- */
- private int batchSize;
- /**
- * queryStatisticsModel to store query statistics object
- */
- QueryStatisticsModel queryStatisticsModel;
-
- public AbstractDetailQueryResultIterator(List<BlockExecutionInfo> infos, QueryModel queryModel,
- ExecutorService execService) {
- String batchSizeString =
- CarbonProperties.getInstance().getProperty(CarbonCommonConstants.DETAIL_QUERY_BATCH_SIZE);
- if (null != batchSizeString) {
- try {
- batchSize = Integer.parseInt(batchSizeString);
- } catch (NumberFormatException ne) {
- LOGGER.error("Invalid inmemory records size. Using default value");
- batchSize = CarbonCommonConstants.DETAIL_QUERY_BATCH_SIZE_DEFAULT;
- }
- } else {
- batchSize = CarbonCommonConstants.DETAIL_QUERY_BATCH_SIZE_DEFAULT;
- }
- this.recorder = queryModel.getStatisticsRecorder();
- this.blockExecutionInfos = infos;
- this.fileReader = FileFactory.getFileHolder(
- FileFactory.getFileType(queryModel.getAbsoluteTableIdentifier().getStorePath()));
- this.execService = execService;
- intialiseInfos();
- initQueryStatiticsModel();
- }
-
- private void intialiseInfos() {
- totalScanTime = System.currentTimeMillis();
- for (BlockExecutionInfo blockInfo : blockExecutionInfos) {
- DataRefNodeFinder finder = new BTreeDataRefNodeFinder(blockInfo.getEachColumnValueSize());
- DataRefNode startDataBlock = finder
- .findFirstDataBlock(blockInfo.getDataBlock().getDataRefNode(), blockInfo.getStartKey());
- while (startDataBlock.nodeNumber() < blockInfo.getStartBlockletIndex()) {
- startDataBlock = startDataBlock.getNextDataRefNode();
- }
-
- long numberOfBlockToScan = blockInfo.getNumberOfBlockletToScan();
- //if number of block is less than 0 then take end block.
- if (numberOfBlockToScan <= 0) {
- DataRefNode endDataBlock = finder
- .findLastDataBlock(blockInfo.getDataBlock().getDataRefNode(), blockInfo.getEndKey());
- numberOfBlockToScan = endDataBlock.nodeNumber() - startDataBlock.nodeNumber() + 1;
- }
- blockInfo.setFirstDataBlock(startDataBlock);
- blockInfo.setNumberOfBlockToScan(numberOfBlockToScan);
- }
- }
-
- @Override public boolean hasNext() {
- if ((dataBlockIterator != null && dataBlockIterator.hasNext()) || nextBatch) {
- return true;
- } else if (blockExecutionInfos.size() > 0) {
- return true;
- } else {
- if (!isStatisticsRecorded) {
- QueryStatistic statistic = new QueryStatistic();
- statistic.addFixedTimeStatistic(QueryStatisticsConstants.SCAN_BLOCKS_TIME,
- System.currentTimeMillis() - totalScanTime);
- recorder.recordStatistics(statistic);
- isStatisticsRecorded = true;
- }
- return false;
- }
- }
-
- protected void updateDataBlockIterator() {
- if (dataBlockIterator == null || !dataBlockIterator.hasNext()) {
- dataBlockIterator = getDataBlockIterator();
- while (dataBlockIterator != null && !dataBlockIterator.hasNext()) {
- dataBlockIterator = getDataBlockIterator();
- }
- }
- }
-
- private DataBlockIteratorImpl getDataBlockIterator() {
- if (blockExecutionInfos.size() > 0) {
- BlockExecutionInfo executionInfo = blockExecutionInfos.get(0);
- blockExecutionInfos.remove(executionInfo);
- queryStatisticsModel.setRecorder(recorder);
- return new DataBlockIteratorImpl(executionInfo, fileReader, batchSize, queryStatisticsModel);
- }
- return null;
- }
-
- protected void initQueryStatiticsModel() {
- this.queryStatisticsModel = new QueryStatisticsModel();
- QueryStatistic queryStatisticTotalBlocklet = new QueryStatistic();
- queryStatisticsModel.getStatisticsTypeAndObjMap()
- .put(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM, queryStatisticTotalBlocklet);
- QueryStatistic queryStatisticValidScanBlocklet = new QueryStatistic();
- queryStatisticsModel.getStatisticsTypeAndObjMap()
- .put(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM, queryStatisticValidScanBlocklet);
- }
-
- public void processNextBatch(CarbonColumnarBatch columnarBatch) {
- throw new UnsupportedOperationException("Please use VectorDetailQueryResultIterator");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/result/iterator/ChunkRowIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/result/iterator/ChunkRowIterator.java b/core/src/main/java/org/apache/carbondata/scan/result/iterator/ChunkRowIterator.java
deleted file mode 100644
index 680b374..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/result/iterator/ChunkRowIterator.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.result.iterator;
-
-import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.scan.result.BatchResult;
-
-/**
- * Iterator over row result
- */
-public class ChunkRowIterator extends CarbonIterator<Object[]> {
-
- /**
- * iterator over chunk result
- */
- private CarbonIterator<BatchResult> iterator;
-
- /**
- * currect chunk
- */
- private BatchResult currentchunk;
-
- public ChunkRowIterator(CarbonIterator<BatchResult> iterator) {
- this.iterator = iterator;
- if (iterator.hasNext()) {
- currentchunk = iterator.next();
- }
- }
-
- /**
- * Returns {@code true} if the iteration has more elements. (In other words,
- * returns {@code true} if {@link #next} would return an element rather than
- * throwing an exception.)
- *
- * @return {@code true} if the iteration has more elements
- */
- @Override public boolean hasNext() {
- if (null != currentchunk) {
- if ((currentchunk.hasNext())) {
- return true;
- } else if (!currentchunk.hasNext()) {
- while (iterator.hasNext()) {
- currentchunk = iterator.next();
- if (currentchunk != null && currentchunk.hasNext()) {
- return true;
- }
- }
- }
- }
- return false;
- }
-
- /**
- * Returns the next element in the iteration.
- *
- * @return the next element in the iteration
- */
- @Override public Object[] next() {
- return currentchunk.next();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/result/iterator/DetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/result/iterator/DetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/scan/result/iterator/DetailQueryResultIterator.java
deleted file mode 100644
index 9afe0f5..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/result/iterator/DetailQueryResultIterator.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.result.iterator;
-
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-
-import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.scan.model.QueryModel;
-import org.apache.carbondata.scan.result.BatchResult;
-
-/**
- * In case of detail query we cannot keep all the records in memory so for
- * executing that query are returning a iterator over block and every time next
- * call will come it will execute the block and return the result
- */
-public class DetailQueryResultIterator extends AbstractDetailQueryResultIterator<BatchResult> {
-
- private final Object lock = new Object();
- private Future<BatchResult> future;
-
- public DetailQueryResultIterator(List<BlockExecutionInfo> infos, QueryModel queryModel,
- ExecutorService execService) {
- super(infos, queryModel, execService);
- }
-
- @Override public BatchResult next() {
- BatchResult result;
- long startTime = System.currentTimeMillis();
- try {
- if (future == null) {
- future = execute();
- }
- result = future.get();
- nextBatch = false;
- if (hasNext()) {
- nextBatch = true;
- future = execute();
- } else {
- fileReader.finish();
- }
- totalScanTime += System.currentTimeMillis() - startTime;
- } catch (Exception ex) {
- try {
- fileReader.finish();
- } finally {
- throw new RuntimeException(ex);
- }
- }
- return result;
- }
-
- private Future<BatchResult> execute() {
- return execService.submit(new Callable<BatchResult>() {
- @Override public BatchResult call() {
- BatchResult batchResult = new BatchResult();
- synchronized (lock) {
- updateDataBlockIterator();
- if (dataBlockIterator != null) {
- batchResult.setRows(dataBlockIterator.next());
- }
- }
- return batchResult;
- }
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/result/iterator/RawResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/result/iterator/RawResultIterator.java b/core/src/main/java/org/apache/carbondata/scan/result/iterator/RawResultIterator.java
deleted file mode 100644
index 0cd2788..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/result/iterator/RawResultIterator.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.result.iterator;
-
-import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.scan.result.BatchResult;
-import org.apache.carbondata.scan.wrappers.ByteArrayWrapper;
-
-/**
- * This is a wrapper iterator over the detail raw query iterator.
- * This iterator will handle the processing of the raw rows.
- * This will handle the batch results and will iterate on the batches and give single row.
- */
-public class RawResultIterator extends CarbonIterator<Object[]> {
-
- private final SegmentProperties sourceSegProperties;
-
- private final SegmentProperties destinationSegProperties;
- /**
- * Iterator of the Batch raw result.
- */
- private CarbonIterator<BatchResult> detailRawQueryResultIterator;
-
- /**
- * Counter to maintain the row counter.
- */
- private int counter = 0;
-
- private Object[] currentConveretedRawRow = null;
-
- /**
- * LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(RawResultIterator.class.getName());
-
- /**
- * batch of the result.
- */
- private BatchResult batch;
-
- public RawResultIterator(CarbonIterator<BatchResult> detailRawQueryResultIterator,
- SegmentProperties sourceSegProperties, SegmentProperties destinationSegProperties) {
- this.detailRawQueryResultIterator = detailRawQueryResultIterator;
- this.sourceSegProperties = sourceSegProperties;
- this.destinationSegProperties = destinationSegProperties;
- }
-
- @Override public boolean hasNext() {
-
- if (null == batch || checkIfBatchIsProcessedCompletely(batch)) {
- if (detailRawQueryResultIterator.hasNext()) {
- batch = null;
- batch = detailRawQueryResultIterator.next();
- counter = 0; // batch changed so reset the counter.
- } else {
- return false;
- }
- }
-
- if (!checkIfBatchIsProcessedCompletely(batch)) {
- return true;
- } else {
- return false;
- }
- }
-
- @Override public Object[] next() {
- if (null == batch) { // for 1st time
- batch = detailRawQueryResultIterator.next();
- }
- if (!checkIfBatchIsProcessedCompletely(batch)) {
- try {
- if(null != currentConveretedRawRow){
- counter++;
- Object[] currentConveretedRawRowTemp = this.currentConveretedRawRow;
- currentConveretedRawRow = null;
- return currentConveretedRawRowTemp;
- }
- return convertRow(batch.getRawRow(counter++));
- } catch (KeyGenException e) {
- LOGGER.error(e.getMessage());
- return null;
- }
- } else { // completed one batch.
- batch = null;
- batch = detailRawQueryResultIterator.next();
- counter = 0;
- }
- try {
- if(null != currentConveretedRawRow){
- counter++;
- Object[] currentConveretedRawRowTemp = this.currentConveretedRawRow;
- currentConveretedRawRow = null;
- return currentConveretedRawRowTemp;
- }
-
- return convertRow(batch.getRawRow(counter++));
- } catch (KeyGenException e) {
- LOGGER.error(e.getMessage());
- return null;
- }
-
- }
-
- /**
- * for fetching the row with out incrementing counter.
- * @return
- */
- public Object[] fetchConverted() throws KeyGenException {
- if(null != currentConveretedRawRow){
- return currentConveretedRawRow;
- }
- if(hasNext())
- {
- Object[] rawRow = batch.getRawRow(counter);
- currentConveretedRawRow = convertRow(rawRow);
- return currentConveretedRawRow;
- }
- else
- {
- return null;
- }
- }
-
- private Object[] convertRow(Object[] rawRow) throws KeyGenException {
- byte[] dims = ((ByteArrayWrapper) rawRow[0]).getDictionaryKey();
- long[] keyArray = sourceSegProperties.getDimensionKeyGenerator().getKeyArray(dims);
- byte[] covertedBytes =
- destinationSegProperties.getDimensionKeyGenerator().generateKey(keyArray);
- ((ByteArrayWrapper) rawRow[0]).setDictionaryKey(covertedBytes);
- return rawRow;
- }
-
- /**
- * To check if the batch is processed completely
- * @param batch
- * @return
- */
- private boolean checkIfBatchIsProcessedCompletely(BatchResult batch){
- if(counter < batch.getSize())
- {
- return false;
- }
- else{
- return true;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/result/iterator/VectorChunkRowIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/result/iterator/VectorChunkRowIterator.java b/core/src/main/java/org/apache/carbondata/scan/result/iterator/VectorChunkRowIterator.java
deleted file mode 100644
index a2ea05b..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/result/iterator/VectorChunkRowIterator.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.result.iterator;
-
-import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.scan.result.vector.CarbonColumnarBatch;
-
-/**
- * Iterator over row result
- */
-public class VectorChunkRowIterator extends CarbonIterator<Object[]> {
-
- /**
- * iterator over chunk result
- */
- private AbstractDetailQueryResultIterator iterator;
-
- /**
- * currect chunk
- */
- private CarbonColumnarBatch columnarBatch;
-
- private int batchSize;
-
- private int currentIndex;
-
- public VectorChunkRowIterator(AbstractDetailQueryResultIterator iterator,
- CarbonColumnarBatch columnarBatch) {
- this.iterator = iterator;
- this.columnarBatch = columnarBatch;
- if (iterator.hasNext()) {
- iterator.processNextBatch(columnarBatch);
- batchSize = columnarBatch.getActualSize();
- currentIndex = 0;
- }
- }
-
- /**
- * Returns {@code true} if the iteration has more elements. (In other words,
- * returns {@code true} if {@link #next} would return an element rather than
- * throwing an exception.)
- *
- * @return {@code true} if the iteration has more elements
- */
- @Override public boolean hasNext() {
- if (currentIndex < batchSize) {
- return true;
- } else {
- while (iterator.hasNext()) {
- columnarBatch.reset();
- iterator.processNextBatch(columnarBatch);
- batchSize = columnarBatch.getActualSize();
- currentIndex = 0;
- if (currentIndex < batchSize) {
- return true;
- }
- }
- }
- return false;
- }
-
- /**
- * Returns the next element in the iteration.
- *
- * @return the next element in the iteration
- */
- @Override public Object[] next() {
- Object[] row = new Object[columnarBatch.columnVectors.length];
- for (int i = 0; i < row.length; i++) {
- row[i] = columnarBatch.columnVectors[i].getData(currentIndex);
- }
- currentIndex++;
- return row;
- }
-
-}
[38/57] [abbrv] incubator-carbondata git commit: move identifier to
metadata package
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
index e2be44d..ebf598f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
@@ -22,7 +22,7 @@ package org.apache.carbondata.core.scan.filter.resolver;
import java.util.ArrayList;
import java.util.List;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.Expression;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
index 90d5db7..88dfab1 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
@@ -25,7 +25,7 @@ import java.util.Comparator;
import java.util.List;
import java.util.SortedMap;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/metadata/FilterResolverMetadata.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/metadata/FilterResolverMetadata.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/metadata/FilterResolverMetadata.java
index 779b532..000825c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/metadata/FilterResolverMetadata.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/metadata/FilterResolverMetadata.java
@@ -18,7 +18,7 @@
*/
package org.apache.carbondata.core.scan.filter.resolver.metadata;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.Expression;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
index 1b295db..41afa4b 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
@@ -26,7 +26,7 @@ import java.util.List;
import java.util.Map;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/service/DictionaryService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/service/DictionaryService.java b/core/src/main/java/org/apache/carbondata/core/service/DictionaryService.java
index 4307604..eabdf82 100644
--- a/core/src/main/java/org/apache/carbondata/core/service/DictionaryService.java
+++ b/core/src/main/java/org/apache/carbondata/core/service/DictionaryService.java
@@ -18,8 +18,8 @@
*/
package org.apache.carbondata.core.service;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader;
import org.apache.carbondata.core.reader.CarbonDictionaryReader;
import org.apache.carbondata.core.reader.sortindex.CarbonDictionarySortIndexReader;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/service/PathService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/service/PathService.java b/core/src/main/java/org/apache/carbondata/core/service/PathService.java
index 21cc349..c5143bf 100644
--- a/core/src/main/java/org/apache/carbondata/core/service/PathService.java
+++ b/core/src/main/java/org/apache/carbondata/core/service/PathService.java
@@ -18,7 +18,7 @@
*/
package org.apache.carbondata.core.service;
-import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.path.CarbonTablePath;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/service/impl/DictionaryFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/service/impl/DictionaryFactory.java b/core/src/main/java/org/apache/carbondata/core/service/impl/DictionaryFactory.java
index d9a6d19..114cb50 100644
--- a/core/src/main/java/org/apache/carbondata/core/service/impl/DictionaryFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/service/impl/DictionaryFactory.java
@@ -18,8 +18,8 @@
*/
package org.apache.carbondata.core.service.impl;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader;
import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReaderImpl;
import org.apache.carbondata.core.reader.CarbonDictionaryReader;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/service/impl/PathFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/service/impl/PathFactory.java b/core/src/main/java/org/apache/carbondata/core/service/impl/PathFactory.java
index 9cd4b41..29b1853 100644
--- a/core/src/main/java/org/apache/carbondata/core/service/impl/PathFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/service/impl/PathFactory.java
@@ -18,7 +18,7 @@
*/
package org.apache.carbondata.core.service.impl;
-import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.service.PathService;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java
index 317d44a..59e8c25 100644
--- a/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java
@@ -24,7 +24,7 @@ import java.util.*;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
@@ -39,7 +39,7 @@ import org.apache.carbondata.core.updatestatus.SegmentStatusManager;
import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.locks.ICarbonLock;
+import org.apache.carbondata.core.updatestatus.locks.ICarbonLock;
/**
@@ -121,12 +121,6 @@ public class CarbonUpdateUtil {
lockStatus = updateLock.lockWithRetries();
if (lockStatus) {
- AbsoluteTableIdentifier absoluteTableIdentifier = table.getAbsoluteTableIdentifier();
-
- CarbonTablePath carbonTablePath = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifier.getStorePath(),
- absoluteTableIdentifier.getCarbonTableIdentifier());
-
// read the existing file if present and update the same.
SegmentUpdateDetails[] oldDetails = segmentUpdateStatusManager
.getUpdateStatusDetails();
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/update/data/BlockletDeleteDeltaCacheLoader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/update/data/BlockletDeleteDeltaCacheLoader.java b/core/src/main/java/org/apache/carbondata/core/update/data/BlockletDeleteDeltaCacheLoader.java
index 2015663..f7c2a2d 100644
--- a/core/src/main/java/org/apache/carbondata/core/update/data/BlockletDeleteDeltaCacheLoader.java
+++ b/core/src/main/java/org/apache/carbondata/core/update/data/BlockletDeleteDeltaCacheLoader.java
@@ -19,7 +19,7 @@ package org.apache.carbondata.core.update.data;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java
index 02c0860..e01ba6d 100644
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java
@@ -26,8 +26,8 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
@@ -35,13 +35,13 @@ import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.load.LoadMetadataDetails;
import org.apache.carbondata.core.update.CarbonUpdateUtil;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.fileoperations.AtomicFileOperations;
-import org.apache.carbondata.fileoperations.AtomicFileOperationsImpl;
-import org.apache.carbondata.fileoperations.FileWriteOperation;
-import org.apache.carbondata.locks.CarbonLockFactory;
-import org.apache.carbondata.locks.CarbonLockUtil;
-import org.apache.carbondata.locks.ICarbonLock;
-import org.apache.carbondata.locks.LockUsage;
+import org.apache.carbondata.core.atomic.AtomicFileOperations;
+import org.apache.carbondata.core.atomic.AtomicFileOperationsImpl;
+import org.apache.carbondata.core.atomic.FileWriteOperation;
+import org.apache.carbondata.core.updatestatus.locks.CarbonLockFactory;
+import org.apache.carbondata.core.updatestatus.locks.CarbonLockUtil;
+import org.apache.carbondata.core.updatestatus.locks.ICarbonLock;
+import org.apache.carbondata.core.updatestatus.locks.LockUsage;
import com.google.gson.Gson;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java
index c283012..4e44661 100644
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java
@@ -25,7 +25,7 @@ import java.util.Map;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
@@ -39,12 +39,12 @@ import org.apache.carbondata.core.update.SegmentUpdateDetails;
import org.apache.carbondata.core.update.TupleIdEnum;
import org.apache.carbondata.core.update.data.UpdateVO;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.fileoperations.AtomicFileOperations;
-import org.apache.carbondata.fileoperations.AtomicFileOperationsImpl;
-import org.apache.carbondata.fileoperations.FileWriteOperation;
-import org.apache.carbondata.locks.CarbonLockFactory;
-import org.apache.carbondata.locks.ICarbonLock;
-import org.apache.carbondata.locks.LockUsage;
+import org.apache.carbondata.core.atomic.AtomicFileOperations;
+import org.apache.carbondata.core.atomic.AtomicFileOperationsImpl;
+import org.apache.carbondata.core.atomic.FileWriteOperation;
+import org.apache.carbondata.core.updatestatus.locks.CarbonLockFactory;
+import org.apache.carbondata.core.updatestatus.locks.ICarbonLock;
+import org.apache.carbondata.core.updatestatus.locks.LockUsage;
import com.google.gson.Gson;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/AbstractCarbonLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/AbstractCarbonLock.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/AbstractCarbonLock.java
new file mode 100644
index 0000000..35f5aee
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/AbstractCarbonLock.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.updatestatus.locks;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+/**
+ * This is the abstract class of the lock implementations.This handles the
+ * retrying part of the locking.
+ */
+abstract class AbstractCarbonLock implements ICarbonLock {
+ private int retryCount;
+
+ private int retryTimeout;
+
+ public abstract boolean lock();
+
+ /**
+ * API for enabling the locking of file with retries.
+ */
+ public boolean lockWithRetries() {
+ try {
+ for (int i = 0; i < retryCount; i++) {
+ if (lock()) {
+ return true;
+ } else {
+ Thread.sleep(retryTimeout * 1000L);
+ }
+ }
+ } catch (InterruptedException e) {
+ return false;
+ }
+ return false;
+ }
+
+ /**
+ * Initializes the retry count and retry timeout.
+ * This will determine how many times to retry to acquire lock and the retry timeout.
+ */
+ protected void initRetry() {
+ String retries = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK);
+ try {
+ retryCount = Integer.parseInt(retries);
+ } catch (NumberFormatException e) {
+ retryCount = CarbonCommonConstants.NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK_DEFAULT;
+ }
+
+ String maxTimeout = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK);
+ try {
+ retryTimeout = Integer.parseInt(maxTimeout);
+ } catch (NumberFormatException e) {
+ retryTimeout = CarbonCommonConstants.MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK_DEFAULT;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockFactory.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockFactory.java
new file mode 100644
index 0000000..0828ada
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockFactory.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.updatestatus.locks;
+
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+/**
+ * This class is a Lock factory class which is used to provide lock objects.
+ * Using this lock object client can request the lock and unlock.
+ */
+public class CarbonLockFactory {
+
+ /**
+ * lockTypeConfigured to check if zookeeper feature is enabled or not for carbon.
+ */
+ private static String lockTypeConfigured;
+
+ static {
+ CarbonLockFactory.getLockTypeConfigured();
+ }
+
+ /**
+ * This method will determine the lock type.
+ *
+ * @param tableIdentifier
+ * @param lockFile
+ * @return
+ */
+ public static ICarbonLock getCarbonLockObj(CarbonTableIdentifier tableIdentifier,
+ String lockFile) {
+ switch (lockTypeConfigured.toUpperCase()) {
+ case CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL:
+ return new LocalFileLock(tableIdentifier, lockFile);
+
+ case CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER:
+ return new ZooKeeperLocking(tableIdentifier, lockFile);
+
+ case CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS:
+ return new HdfsFileLock(tableIdentifier, lockFile);
+
+ default:
+ throw new UnsupportedOperationException("Not supported the lock type");
+ }
+ }
+
+ /**
+ *
+ * @param locFileLocation
+ * @param lockFile
+ * @return carbon lock
+ */
+ public static ICarbonLock getCarbonLockObj(String locFileLocation, String lockFile) {
+ switch (lockTypeConfigured.toUpperCase()) {
+ case CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL:
+ return new LocalFileLock(locFileLocation, lockFile);
+
+ case CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER:
+ return new ZooKeeperLocking(locFileLocation, lockFile);
+
+ case CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS:
+ return new HdfsFileLock(locFileLocation, lockFile);
+
+ default:
+ throw new UnsupportedOperationException("Not supported the lock type");
+ }
+ }
+
+ /**
+ * This method will set the zookeeper status whether zookeeper to be used for locking or not.
+ */
+ private static void getLockTypeConfigured() {
+ lockTypeConfigured = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.LOCK_TYPE, CarbonCommonConstants.LOCK_TYPE_DEFAULT);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockUtil.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockUtil.java
new file mode 100644
index 0000000..b73bedd
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockUtil.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.updatestatus.locks;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+
+/**
+ * This class contains all carbon lock utilities
+ */
+public class CarbonLockUtil {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CarbonLockUtil.class.getName());
+
+ /**
+ * unlocks given file
+ *
+ * @param carbonLock
+ */
+ public static void fileUnlock(ICarbonLock carbonLock, String locktype) {
+ if (carbonLock.unlock()) {
+ if (locktype.equals(LockUsage.METADATA_LOCK)) {
+ LOGGER.info("Metadata lock has been successfully released");
+ } else if (locktype.equals(LockUsage.TABLE_STATUS_LOCK)) {
+ LOGGER.info("Table status lock has been successfully released");
+ }
+ else if (locktype.equals(LockUsage.CLEAN_FILES_LOCK)) {
+ LOGGER.info("Clean files lock has been successfully released");
+ }
+ else if (locktype.equals(LockUsage.DELETE_SEGMENT_LOCK)) {
+ LOGGER.info("Delete segments lock has been successfully released");
+ }
+ } else {
+ if (locktype.equals(LockUsage.METADATA_LOCK)) {
+ LOGGER.error("Not able to release the metadata lock");
+ } else if (locktype.equals(LockUsage.TABLE_STATUS_LOCK)) {
+ LOGGER.error("Not able to release the table status lock");
+ }
+ else if (locktype.equals(LockUsage.CLEAN_FILES_LOCK)) {
+ LOGGER.info("Not able to release the clean files lock");
+ }
+ else if (locktype.equals(LockUsage.DELETE_SEGMENT_LOCK)) {
+ LOGGER.info("Not able to release the delete segments lock");
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/HdfsFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/HdfsFileLock.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/HdfsFileLock.java
new file mode 100644
index 0000000..1d1de81
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/HdfsFileLock.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.updatestatus.locks;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * This class is used to handle the HDFS File locking.
+ * This is acheived using the concept of acquiring the data out stream using Append option.
+ */
+class HdfsFileLock extends AbstractCarbonLock {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(HdfsFileLock.class.getName());
+ /**
+ * location hdfs file location
+ */
+ private String location;
+
+ private DataOutputStream dataOutputStream;
+
+ public static String tmpPath;
+
+ static {
+ Configuration conf = new Configuration(true);
+ String hdfsPath = conf.get(CarbonCommonConstants.FS_DEFAULT_FS);
+ // By default, we put the hdfs lock meta file for one table inside this table's store folder.
+ // If can not get the STORE_LOCATION, then use hadoop.tmp.dir .
+ tmpPath = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION,
+ System.getProperty(CarbonCommonConstants.HDFS_TEMP_LOCATION));
+ if (!tmpPath.startsWith(CarbonCommonConstants.HDFSURL_PREFIX)) {
+ tmpPath = hdfsPath + tmpPath;
+ }
+ }
+
+ /**
+ * @param lockFileLocation
+ * @param lockFile
+ */
+ public HdfsFileLock(String lockFileLocation, String lockFile) {
+ this.location = tmpPath + CarbonCommonConstants.FILE_SEPARATOR + lockFileLocation
+ + CarbonCommonConstants.FILE_SEPARATOR + lockFile;
+ LOGGER.info("HDFS lock path:"+this.location);
+ initRetry();
+ }
+
+ /**
+ * @param tableIdentifier
+ * @param lockFile
+ */
+ public HdfsFileLock(CarbonTableIdentifier tableIdentifier, String lockFile) {
+ this(tableIdentifier.getDatabaseName() + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier
+ .getTableName(), lockFile);
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.carbondata.core.locks.ICarbonLock#lock()
+ */
+ @Override public boolean lock() {
+ try {
+ if (!FileFactory.isFileExist(location, FileFactory.getFileType(location))) {
+ FileFactory.createNewLockFile(location, FileFactory.getFileType(location));
+ }
+ dataOutputStream =
+ FileFactory.getDataOutputStreamUsingAppend(location, FileFactory.getFileType(location));
+
+ return true;
+
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.carbondata.core.locks.ICarbonLock#unlock()
+ */
+ @Override
+ public boolean unlock() {
+ if (null != dataOutputStream) {
+ try {
+ dataOutputStream.close();
+ } catch (IOException e) {
+ return false;
+ } finally {
+ CarbonFile carbonFile =
+ FileFactory.getCarbonFile(location, FileFactory.getFileType(location));
+ if (carbonFile.exists()) {
+ if (carbonFile.delete()) {
+ LOGGER.info("Deleted the lock file " + location);
+ } else {
+ LOGGER.error("Not able to delete the lock file " + location);
+ }
+ } else {
+ LOGGER.error("Not able to delete the lock file because "
+ + "it is not existed in location " + location);
+ }
+ }
+ }
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ICarbonLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ICarbonLock.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ICarbonLock.java
new file mode 100644
index 0000000..e269c86
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ICarbonLock.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.updatestatus.locks;
+
+/**
+ * Carbon Lock Interface which handles the locking and unlocking.
+ */
+public interface ICarbonLock {
+
+ /**
+ * Does the unlocking of the acquired lock.
+ *
+ * @return
+ */
+ boolean unlock();
+
+ /**
+ * This will acquire the lock and if it doesnt get then it will retry after the confiured time.
+ *
+ * @return
+ */
+ boolean lockWithRetries();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLock.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLock.java
new file mode 100644
index 0000000..cdee312
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLock.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.updatestatus.locks;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.channels.OverlappingFileLockException;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
+
+/**
+ * This class handles the file locking in the local file system.
+ * This will be handled using the file channel lock API.
+ */
+class LocalFileLock extends AbstractCarbonLock {
+ /**
+ * location is the location of the lock file.
+ */
+ private String location;
+
+ /**
+ * fileOutputStream of the local lock file
+ */
+ private FileOutputStream fileOutputStream;
+
+ /**
+ * channel is the FileChannel of the lock file.
+ */
+ private FileChannel channel;
+
+ /**
+ * fileLock NIO FileLock Object
+ */
+ private FileLock fileLock;
+
+ /**
+ * lock file
+ */
+ private String lockFile;
+
+ public static final String tmpPath;
+
+ private String lockFilePath;
+
+ /**
+ * LOGGER for logging the messages.
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(LocalFileLock.class.getName());
+
+ static {
+ tmpPath = System.getProperty("java.io.tmpdir");
+ }
+
+ /**
+ * @param lockFileLocation
+ * @param lockFile
+ */
+ public LocalFileLock(String lockFileLocation, String lockFile) {
+ this.location = tmpPath + CarbonCommonConstants.FILE_SEPARATOR + lockFileLocation;
+ this.lockFile = lockFile;
+ initRetry();
+ }
+
+ /**
+ * @param tableIdentifier
+ * @param lockFile
+ */
+ public LocalFileLock(CarbonTableIdentifier tableIdentifier, String lockFile) {
+ this(tableIdentifier.getDatabaseName() + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier
+ .getTableName(), lockFile);
+ initRetry();
+ }
+
+ /**
+ * Lock API for locking of the file channel of the lock file.
+ *
+ * @return
+ */
+ @Override public boolean lock() {
+ try {
+ if (!FileFactory.isFileExist(location, FileFactory.getFileType(tmpPath))) {
+ FileFactory.mkdirs(location, FileFactory.getFileType(tmpPath));
+ }
+ lockFilePath = location + CarbonCommonConstants.FILE_SEPARATOR +
+ lockFile;
+ if (!FileFactory.isFileExist(lockFilePath, FileFactory.getFileType(location))) {
+ FileFactory.createNewLockFile(lockFilePath, FileFactory.getFileType(location));
+ }
+
+ fileOutputStream = new FileOutputStream(lockFilePath);
+ channel = fileOutputStream.getChannel();
+ try {
+ fileLock = channel.tryLock();
+ } catch (OverlappingFileLockException e) {
+ return false;
+ }
+ if (null != fileLock) {
+ return true;
+ } else {
+ return false;
+ }
+ } catch (IOException e) {
+ return false;
+ }
+
+ }
+
+ /**
+ * Unlock API for unlocking of the acquired lock.
+ *
+ * @return
+ */
+ @Override public boolean unlock() {
+ boolean status;
+ try {
+ if (null != fileLock) {
+ fileLock.release();
+ }
+ status = true;
+ } catch (IOException e) {
+ status = false;
+ } finally {
+ if (null != fileOutputStream) {
+ try {
+ fileOutputStream.close();
+ // deleting the lock file after releasing the lock.
+ if (FileFactory.getCarbonFile(lockFilePath, FileFactory.getFileType(lockFilePath))
+ .delete()) {
+ LOGGER.info("Successfully deleted the lock file " + lockFilePath);
+ } else {
+ LOGGER.error("Not able to delete the lock file " + lockFilePath);
+ }
+ } catch (IOException e) {
+ LOGGER.error(e.getMessage());
+ }
+ }
+ }
+ return status;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LockUsage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LockUsage.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LockUsage.java
new file mode 100644
index 0000000..5a2ffe7
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LockUsage.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.updatestatus.locks;
+
+/**
+ * This enum is used to define the usecase of the lock.
+ * Each enum value is one specific lock case.
+ */
+public class LockUsage {
+ public static final String LOCK = ".lock";
+ public static final String METADATA_LOCK = "meta.lock";
+ public static final String COMPACTION_LOCK = "compaction.lock";
+ public static final String SYSTEMLEVEL_COMPACTION_LOCK = "system_level_compaction.lock";
+ public static final String TABLE_STATUS_LOCK = "tablestatus.lock";
+ public static final String TABLE_UPDATE_STATUS_LOCK = "tableupdatestatus.lock";
+ public static final String DELETE_SEGMENT_LOCK = "delete_segment.lock";
+ public static final String CLEAN_FILES_LOCK = "clean_files.lock";
+ public static final String DROP_TABLE_LOCK = "droptable.lock";
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZooKeeperLocking.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZooKeeperLocking.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZooKeeperLocking.java
new file mode 100644
index 0000000..2d742cf
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZooKeeperLocking.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.updatestatus.locks;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.ZooKeeper;
+
+/**
+ * For Handling the zookeeper locking implementation
+ */
+class ZooKeeperLocking extends AbstractCarbonLock {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(ZooKeeperLocking.class.getName());
+
+ /**
+ * zk is the zookeeper client instance
+ */
+ private static ZooKeeper zk;
+
+ /**
+ * zooKeeperLocation is the location in the zoo keeper file system where the locks will be
+ * maintained.
+ */
+ private static final String zooKeeperLocation = CarbonCommonConstants.ZOOKEEPER_LOCATION;
+
+ /**
+ * Unique folder for each table with DatabaseName_TableName
+ */
+ private final String tableIdFolder;
+
+ /**
+ * lockName is the name of the lock to use. This name should be same for every process that want
+ * to share the same lock
+ */
+ private String lockName;
+
+ /**
+ * lockPath is the unique path created for the each instance of the carbon lock.
+ */
+ private String lockPath;
+
+ private String lockTypeFolder;
+
+ public ZooKeeperLocking(CarbonTableIdentifier tableIdentifier, String lockFile) {
+ this(tableIdentifier.getDatabaseName() + File.separator + tableIdentifier.getTableName(),
+ lockFile);
+ }
+
+ /**
+ * @param lockLocation
+ * @param lockFile
+ */
+ public ZooKeeperLocking(String lockLocation, String lockFile) {
+ this.lockName = lockFile;
+ this.tableIdFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + lockLocation;
+
+ String zooKeeperUrl =
+ CarbonProperties.getInstance().getProperty(CarbonCommonConstants.ZOOKEEPER_URL);
+ zk = ZookeeperInit.getInstance(zooKeeperUrl).getZookeeper();
+
+ this.lockTypeFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + lockLocation
+ + CarbonCommonConstants.FILE_SEPARATOR + lockFile;
+ try {
+ createBaseNode();
+ // if exists returns null then path doesnt exist. so creating.
+ if (null == zk.exists(this.tableIdFolder, true)) {
+ createRecursivly(this.tableIdFolder);
+ }
+ // if exists returns null then path doesnt exist. so creating.
+ if (null == zk.exists(this.lockTypeFolder, true)) {
+ zk.create(this.lockTypeFolder, new byte[1], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+ }
+ } catch (KeeperException | InterruptedException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ initRetry();
+ }
+
+ /**
+ * Creating a znode in which all the znodes (lock files )are maintained.
+ */
+ private void createBaseNode() throws KeeperException, InterruptedException {
+ if (null == zk.exists(zooKeeperLocation, true)) {
+ // creating a znode in which all the znodes (lock files )are maintained.
+ zk.create(zooKeeperLocation, new byte[1], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+ }
+ }
+
+ /**
+ * Create zookeepr node if not exist
+ * @param path
+ * @throws KeeperException
+ * @throws InterruptedException
+ */
+ private void createRecursivly(String path) throws KeeperException, InterruptedException {
+ try {
+ if (zk.exists(path, true) == null && path.length() > 0) {
+ String temp = path.substring(0, path.lastIndexOf(File.separator));
+ createRecursivly(temp);
+ zk.create(path, null, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+ } else {
+ return;
+ }
+ } catch (KeeperException e) {
+ throw e;
+ } catch (InterruptedException e) {
+ throw e;
+ }
+
+ }
+ /**
+ * Handling of the locking mechanism using zoo keeper.
+ */
+ @Override public boolean lock() {
+ try {
+ // create the lock file with lockName.
+ lockPath =
+ zk.create(this.lockTypeFolder + CarbonCommonConstants.FILE_SEPARATOR + lockName, null,
+ Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL);
+
+ // get the children present in zooKeeperLocation.
+ List<String> nodes = zk.getChildren(this.lockTypeFolder, null);
+
+ // sort the childrens
+ Collections.sort(nodes);
+
+ // here the logic is , for each lock request zookeeper will create a file ending with
+ // incremental digits.
+ // so first request will be 00001 next is 00002 and so on.
+ // if the current request is 00002 and already one previous request(00001) is present then get
+ // children will give both nodes.
+ // after the sort we are checking if the lock path is first or not .if it is first then lock
+ // has been acquired.
+
+ if (lockPath.endsWith(nodes.get(0))) {
+ return true;
+ } else {
+ // if locking failed then deleting the created lock as next time again new lock file will be
+ // created.
+ zk.delete(lockPath, -1);
+ return false;
+ }
+ } catch (KeeperException | InterruptedException e) {
+ LOGGER.error(e, e.getMessage());
+ return false;
+ }
+ }
+
+ /**
+ * @return status where lock file is unlocked or not.
+ */
+ @Override public boolean unlock() {
+ try {
+ // exists will return null if the path doesn't exists.
+ if (null != zk.exists(lockPath, true)) {
+ zk.delete(lockPath, -1);
+ lockPath = null;
+ }
+ } catch (KeeperException | InterruptedException e) {
+ LOGGER.error(e, e.getMessage());
+ return false;
+ }
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZookeeperInit.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZookeeperInit.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZookeeperInit.java
new file mode 100644
index 0000000..391e174
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZookeeperInit.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.updatestatus.locks;
+
+import java.io.IOException;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+
+/**
+ * This is a singleton class for initialization of zookeeper client.
+ */
+public class ZookeeperInit {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(ZookeeperInit.class.getName());
+
+ private static ZookeeperInit zooKeeperInit;
+ /**
+ * zk is the zookeeper client instance
+ */
+ private ZooKeeper zk;
+
+ private ZookeeperInit(String zooKeeperUrl) {
+
+ int sessionTimeOut = 100000;
+ try {
+ zk = new ZooKeeper(zooKeeperUrl, sessionTimeOut, new DummyWatcher());
+
+ } catch (IOException e) {
+ LOGGER.error(e.getMessage());
+ }
+
+ }
+
+ public static ZookeeperInit getInstance(String zooKeeperUrl) {
+
+ if (null == zooKeeperInit) {
+ synchronized (ZookeeperInit.class) {
+ if (null == zooKeeperInit) {
+ LOGGER.info("Initiating Zookeeper client.");
+ zooKeeperInit = new ZookeeperInit(zooKeeperUrl);
+ }
+ }
+ }
+ return zooKeeperInit;
+
+ }
+
+ public static ZookeeperInit getInstance() {
+ return zooKeeperInit;
+ }
+
+ public ZooKeeper getZookeeper() {
+ return zk;
+ }
+
+ private static class DummyWatcher implements Watcher {
+ public void process(WatchedEvent event) {
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
index 59305b1..fa8410a 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
@@ -24,7 +24,7 @@ import java.util.ArrayList;
import java.util.BitSet;
import java.util.List;
-import org.apache.carbondata.core.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.datastore.block.BlockInfo;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
index 360fd6f..79d5b7b 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
@@ -30,7 +30,7 @@ import java.util.Set;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.metadata.index.BlockIndexInfo;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index 47c9926..95d099a 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -27,7 +27,7 @@ import java.util.Properties;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
public final class CarbonProperties {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 2a4fb1b..34f1ac9 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -43,7 +43,7 @@ import java.util.Set;
import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
index 4d9e5b8..eadac33 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
@@ -23,7 +23,7 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import org.apache.carbondata.core.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
index 8ca8ddd..0808cea 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
@@ -22,7 +22,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.carbondata.core.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterFactory.java b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterFactory.java
index 19df747..92a507a 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverterFactory.java
@@ -18,7 +18,7 @@
*/
package org.apache.carbondata.core.util;
-import org.apache.carbondata.core.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
/**
* Factory class to get the thrift reader object based on version
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
index 9fe894a..e024653 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
@@ -28,8 +28,8 @@ import java.util.List;
import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
index 9f9d7ab..1d37b5b 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
@@ -26,8 +26,8 @@ import java.util.List;
import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/fileoperations/AtomicFileOperations.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/fileoperations/AtomicFileOperations.java b/core/src/main/java/org/apache/carbondata/fileoperations/AtomicFileOperations.java
deleted file mode 100644
index a1c7a2e..0000000
--- a/core/src/main/java/org/apache/carbondata/fileoperations/AtomicFileOperations.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.fileoperations;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-public interface AtomicFileOperations {
-
- DataInputStream openForRead() throws IOException;
-
- void close() throws IOException;
-
- DataOutputStream openForWrite(FileWriteOperation operation) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/fileoperations/AtomicFileOperationsImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/fileoperations/AtomicFileOperationsImpl.java b/core/src/main/java/org/apache/carbondata/fileoperations/AtomicFileOperationsImpl.java
deleted file mode 100644
index 1bb8ba3..0000000
--- a/core/src/main/java/org/apache/carbondata/fileoperations/AtomicFileOperationsImpl.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.fileoperations;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
-import org.apache.carbondata.core.datastorage.impl.FileFactory.FileType;
-
-public class AtomicFileOperationsImpl implements AtomicFileOperations {
-
- private String filePath;
-
- private FileType fileType;
-
- private String tempWriteFilePath;
-
- private DataOutputStream dataOutStream;
-
- public AtomicFileOperationsImpl(String filePath, FileType fileType) {
- this.filePath = filePath;
-
- this.fileType = fileType;
- }
-
- @Override public DataInputStream openForRead() throws IOException {
- return FileFactory.getDataInputStream(filePath, fileType);
- }
-
- @Override public DataOutputStream openForWrite(FileWriteOperation operation) throws IOException {
-
- filePath = filePath.replace("\\", "/");
-
- tempWriteFilePath = filePath + CarbonCommonConstants.TEMPWRITEFILEEXTENSION;
-
- if (FileFactory.isFileExist(tempWriteFilePath, fileType)) {
- FileFactory.getCarbonFile(tempWriteFilePath, fileType).delete();
- }
-
- FileFactory.createNewFile(tempWriteFilePath, fileType);
-
- dataOutStream = FileFactory.getDataOutputStream(tempWriteFilePath, fileType);
-
- return dataOutStream;
-
- }
-
- /* (non-Javadoc)
- * @see com.huawei.unibi.carbon.datastorage.store.fileperations.AtomicFileOperations#close()
- */
- @Override public void close() throws IOException {
-
- if (null != dataOutStream) {
- dataOutStream.close();
-
- CarbonFile tempFile = FileFactory.getCarbonFile(tempWriteFilePath, fileType);
-
- if (!tempFile.renameForce(filePath)) {
- throw new IOException("temporary file renaming failed, src="
- + tempFile.getPath() + ", dest=" + filePath);
- }
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/fileoperations/FileWriteOperation.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/fileoperations/FileWriteOperation.java b/core/src/main/java/org/apache/carbondata/fileoperations/FileWriteOperation.java
deleted file mode 100644
index 81502f2..0000000
--- a/core/src/main/java/org/apache/carbondata/fileoperations/FileWriteOperation.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.fileoperations;
-
-public enum FileWriteOperation {
-
- APPEND, OVERWRITE
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/locks/AbstractCarbonLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/locks/AbstractCarbonLock.java b/core/src/main/java/org/apache/carbondata/locks/AbstractCarbonLock.java
deleted file mode 100644
index 934f447..0000000
--- a/core/src/main/java/org/apache/carbondata/locks/AbstractCarbonLock.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.locks;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.util.CarbonProperties;
-
-/**
- * This is the abstract class of the lock implementations.This handles the
- * retrying part of the locking.
- */
-public abstract class AbstractCarbonLock implements ICarbonLock {
- private int retryCount;
-
- private int retryTimeout;
-
- public abstract boolean lock();
-
- /**
- * API for enabling the locking of file with retries.
- */
- public boolean lockWithRetries() {
- try {
- for (int i = 0; i < retryCount; i++) {
- if (lock()) {
- return true;
- } else {
- Thread.sleep(retryTimeout * 1000L);
- }
- }
- } catch (InterruptedException e) {
- return false;
- }
- return false;
- }
-
- /**
- * Initializes the retry count and retry timeout.
- * This will determine how many times to retry to acquire lock and the retry timeout.
- */
- protected void initRetry() {
- String retries = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK);
- try {
- retryCount = Integer.parseInt(retries);
- } catch (NumberFormatException e) {
- retryCount = CarbonCommonConstants.NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK_DEFAULT;
- }
-
- String maxTimeout = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK);
- try {
- retryTimeout = Integer.parseInt(maxTimeout);
- } catch (NumberFormatException e) {
- retryTimeout = CarbonCommonConstants.MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK_DEFAULT;
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/locks/CarbonLockFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/locks/CarbonLockFactory.java b/core/src/main/java/org/apache/carbondata/locks/CarbonLockFactory.java
deleted file mode 100644
index d4d60a8..0000000
--- a/core/src/main/java/org/apache/carbondata/locks/CarbonLockFactory.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.locks;
-
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.util.CarbonProperties;
-
-/**
- * This class is a Lock factory class which is used to provide lock objects.
- * Using this lock object client can request the lock and unlock.
- */
-public class CarbonLockFactory {
-
- /**
- * lockTypeConfigured to check if zookeeper feature is enabled or not for carbon.
- */
- private static String lockTypeConfigured;
-
- static {
- CarbonLockFactory.getLockTypeConfigured();
- }
-
- /**
- * This method will determine the lock type.
- *
- * @param tableIdentifier
- * @param lockFile
- * @return
- */
- public static ICarbonLock getCarbonLockObj(CarbonTableIdentifier tableIdentifier,
- String lockFile) {
- switch (lockTypeConfigured.toUpperCase()) {
- case CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL:
- return new LocalFileLock(tableIdentifier, lockFile);
-
- case CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER:
- return new ZooKeeperLocking(tableIdentifier, lockFile);
-
- case CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS:
- return new HdfsFileLock(tableIdentifier, lockFile);
-
- default:
- throw new UnsupportedOperationException("Not supported the lock type");
- }
- }
-
- /**
- *
- * @param locFileLocation
- * @param lockFile
- * @return carbon lock
- */
- public static ICarbonLock getCarbonLockObj(String locFileLocation, String lockFile) {
- switch (lockTypeConfigured.toUpperCase()) {
- case CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL:
- return new LocalFileLock(locFileLocation, lockFile);
-
- case CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER:
- return new ZooKeeperLocking(locFileLocation, lockFile);
-
- case CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS:
- return new HdfsFileLock(locFileLocation, lockFile);
-
- default:
- throw new UnsupportedOperationException("Not supported the lock type");
- }
- }
-
- /**
- * This method will set the zookeeper status whether zookeeper to be used for locking or not.
- */
- private static void getLockTypeConfigured() {
- lockTypeConfigured = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.LOCK_TYPE, CarbonCommonConstants.LOCK_TYPE_DEFAULT);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/locks/CarbonLockUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/locks/CarbonLockUtil.java b/core/src/main/java/org/apache/carbondata/locks/CarbonLockUtil.java
deleted file mode 100644
index 181e92b..0000000
--- a/core/src/main/java/org/apache/carbondata/locks/CarbonLockUtil.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.locks;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-
-/**
- * This class contains all carbon lock utilities
- */
-public class CarbonLockUtil {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CarbonLockUtil.class.getName());
-
- /**
- * unlocks given file
- *
- * @param carbonLock
- */
- public static void fileUnlock(ICarbonLock carbonLock, String locktype) {
- if (carbonLock.unlock()) {
- if (locktype.equals(LockUsage.METADATA_LOCK)) {
- LOGGER.info("Metadata lock has been successfully released");
- } else if (locktype.equals(LockUsage.TABLE_STATUS_LOCK)) {
- LOGGER.info("Table status lock has been successfully released");
- }
- else if (locktype.equals(LockUsage.CLEAN_FILES_LOCK)) {
- LOGGER.info("Clean files lock has been successfully released");
- }
- else if (locktype.equals(LockUsage.DELETE_SEGMENT_LOCK)) {
- LOGGER.info("Delete segments lock has been successfully released");
- }
- } else {
- if (locktype.equals(LockUsage.METADATA_LOCK)) {
- LOGGER.error("Not able to release the metadata lock");
- } else if (locktype.equals(LockUsage.TABLE_STATUS_LOCK)) {
- LOGGER.error("Not able to release the table status lock");
- }
- else if (locktype.equals(LockUsage.CLEAN_FILES_LOCK)) {
- LOGGER.info("Not able to release the clean files lock");
- }
- else if (locktype.equals(LockUsage.DELETE_SEGMENT_LOCK)) {
- LOGGER.info("Not able to release the delete segments lock");
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/locks/HdfsFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/locks/HdfsFileLock.java b/core/src/main/java/org/apache/carbondata/locks/HdfsFileLock.java
deleted file mode 100644
index d61df15..0000000
--- a/core/src/main/java/org/apache/carbondata/locks/HdfsFileLock.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.locks;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
-import org.apache.carbondata.core.util.CarbonProperties;
-
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * This class is used to handle the HDFS File locking.
- * This is acheived using the concept of acquiring the data out stream using Append option.
- */
-public class HdfsFileLock extends AbstractCarbonLock {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(HdfsFileLock.class.getName());
- /**
- * location hdfs file location
- */
- private String location;
-
- private DataOutputStream dataOutputStream;
-
- public static String tmpPath;
-
- static {
- Configuration conf = new Configuration(true);
- String hdfsPath = conf.get(CarbonCommonConstants.FS_DEFAULT_FS);
- // By default, we put the hdfs lock meta file for one table inside this table's store folder.
- // If can not get the STORE_LOCATION, then use hadoop.tmp.dir .
- tmpPath = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION,
- System.getProperty(CarbonCommonConstants.HDFS_TEMP_LOCATION));
- if (!tmpPath.startsWith(CarbonCommonConstants.HDFSURL_PREFIX)) {
- tmpPath = hdfsPath + tmpPath;
- }
- }
-
- /**
- * @param lockFileLocation
- * @param lockFile
- */
- public HdfsFileLock(String lockFileLocation, String lockFile) {
- this.location = tmpPath + CarbonCommonConstants.FILE_SEPARATOR + lockFileLocation
- + CarbonCommonConstants.FILE_SEPARATOR + lockFile;
- LOGGER.info("HDFS lock path:"+this.location);
- initRetry();
- }
-
- /**
- * @param tableIdentifier
- * @param lockFile
- */
- public HdfsFileLock(CarbonTableIdentifier tableIdentifier, String lockFile) {
- this(tableIdentifier.getDatabaseName() + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier
- .getTableName(), lockFile);
- }
-
- /* (non-Javadoc)
- * @see org.apache.carbondata.core.locks.ICarbonLock#lock()
- */
- @Override public boolean lock() {
- try {
- if (!FileFactory.isFileExist(location, FileFactory.getFileType(location))) {
- FileFactory.createNewLockFile(location, FileFactory.getFileType(location));
- }
- dataOutputStream =
- FileFactory.getDataOutputStreamUsingAppend(location, FileFactory.getFileType(location));
-
- return true;
-
- } catch (IOException e) {
- return false;
- }
- }
-
- /* (non-Javadoc)
- * @see org.apache.carbondata.core.locks.ICarbonLock#unlock()
- */
- @Override
- public boolean unlock() {
- if (null != dataOutputStream) {
- try {
- dataOutputStream.close();
- } catch (IOException e) {
- return false;
- } finally {
- CarbonFile carbonFile =
- FileFactory.getCarbonFile(location, FileFactory.getFileType(location));
- if (carbonFile.exists()) {
- if (carbonFile.delete()) {
- LOGGER.info("Deleted the lock file " + location);
- } else {
- LOGGER.error("Not able to delete the lock file " + location);
- }
- } else {
- LOGGER.error("Not able to delete the lock file because "
- + "it is not existed in location " + location);
- }
- }
- }
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/locks/ICarbonLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/locks/ICarbonLock.java b/core/src/main/java/org/apache/carbondata/locks/ICarbonLock.java
deleted file mode 100644
index fbb2030..0000000
--- a/core/src/main/java/org/apache/carbondata/locks/ICarbonLock.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.locks;
-
-/**
- * Carbon Lock Interface which handles the locking and unlocking.
- */
-public interface ICarbonLock {
-
- /**
- * Does the unlocking of the acquired lock.
- *
- * @return
- */
- boolean unlock();
-
- /**
- * This will acquire the lock and if it doesnt get then it will retry after the confiured time.
- *
- * @return
- */
- boolean lockWithRetries();
-
-}
[10/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java
deleted file mode 100644
index 8277f91..0000000
--- a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileFactoryImplUnitTest.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.carbon.datastorage.filesystem.store.impl;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static junit.framework.TestCase.assertNotNull;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-public class FileFactoryImplUnitTest {
-
- private static String filePath;
-
- @AfterClass
- public static void tearDown() {
- File file = new File(filePath);
- if (file.exists()) {
- file.delete();
- }
-
- File file1 = new File(".TestFileFactory.carbondata.crc");
- if (file1.exists()) {
- file1.delete();
- }
- }
-
- @BeforeClass public static void setUp() {
- filePath = "TestFileFactory.carbondata";
- }
-
- @Test public void testFileExistsForVIEWFSType() throws IOException {
- FileFactory.isFileExist("fakefilePath", FileFactory.FileType.VIEWFS);
- }
-
- @Test public void testFileExistsForDefaultType() throws IOException {
- FileFactory.isFileExist("fakefilePath", FileFactory.FileType.LOCAL);
- }
-
- @Test public void testFileExistsForDefaultTypeWithPerformFileCheck() throws IOException {
- assertTrue(FileFactory.isFileExist(filePath, FileFactory.FileType.LOCAL, true));
- }
-
- @Test public void testFileExistsForDefaultTypeWithOutPerformFileCheck() throws IOException {
- assertFalse(FileFactory.isFileExist("fakefilePath", FileFactory.FileType.LOCAL, false));
- }
-
- @Test public void testFileExistsForVIEWFSTypeWithPerformFileCheck() throws IOException {
- assertTrue(FileFactory.isFileExist(filePath, FileFactory.FileType.VIEWFS, true));
- }
-
- @Test public void testFileExistsForVIEWFSTypeWithOutPerformFileCheck() throws IOException {
- assertFalse(FileFactory.isFileExist("fakefilePath", FileFactory.FileType.VIEWFS, false));
- }
-
- @Test public void testCreateNewFileWithDefaultFileType() throws IOException {
- tearDown();
- assertTrue(FileFactory.createNewFile(filePath, FileFactory.FileType.LOCAL));
- }
-
- @Test public void testCreateNewLockFileWithDefaultFileType() throws IOException {
- tearDown();
- assertTrue(FileFactory.createNewLockFile(filePath, FileFactory.FileType.LOCAL));
- }
-
- @Test public void testCreateNewLockFileWithViewFsFileType() throws IOException {
- tearDown();
- assertTrue(FileFactory.createNewLockFile(filePath, FileFactory.FileType.VIEWFS));
- }
-
- @Test public void testCreateNewLockFileWithViewFsFileTypeWhenFileExists() throws IOException {
- assertFalse(FileFactory.createNewLockFile(filePath, FileFactory.FileType.VIEWFS));
- }
-
- @Test public void testCreateNewFileWithDefaultFileTypeWhenFileExists() throws IOException {
- assertFalse(FileFactory.createNewFile(filePath, FileFactory.FileType.LOCAL));
- }
-
- @Test public void testCreateNewFileWithVIEWFSFileType() throws IOException {
- File file = new File(filePath);
- if (file.exists()) {
- file.delete();
- }
- assertTrue(FileFactory.createNewFile(filePath, FileFactory.FileType.VIEWFS));
- }
-
- @Test public void testCreateNewFileWithVIEWFSFileTypeWhenFileExists() throws IOException {
- assertFalse(FileFactory.createNewFile(filePath, FileFactory.FileType.VIEWFS));
- }
-
- @Test public void testMkDirWithVIEWFSFileType() throws IOException {
- new MockUp<FileSystem>() {
- @SuppressWarnings("unused") @Mock public boolean mkdirs(Path file) throws IOException {
- {
- return true;
- }
- }
- };
- tearDown();
- assertTrue(FileFactory.mkdirs(filePath, FileFactory.FileType.VIEWFS));
- }
-
- @Test public void testGetDataOutputStreamUsingAppendeForException() {
- try {
- FileFactory.getDataOutputStreamUsingAppend(filePath, FileFactory.FileType.VIEWFS);
- } catch (Exception exception) {
- assertEquals("Not supported", exception.getMessage());
- }
- }
-
- @Test public void getDataOutputStreamForVIEWFSType() throws IOException {
- assertNotNull(FileFactory.getDataOutputStream(filePath, FileFactory.FileType.VIEWFS));
- }
-
- @Test public void getDataOutputStreamForLocalType() throws IOException {
- assertNotNull(FileFactory.getDataOutputStream(filePath, FileFactory.FileType.LOCAL));
- }
-
- @Test public void testGetCarbonFile() throws IOException {
- FileFactory.getDataOutputStream(filePath, FileFactory.FileType.VIEWFS);
- assertNotNull(FileFactory.getCarbonFile(filePath, FileFactory.FileType.HDFS));
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileHolderImplUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileHolderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileHolderImplUnitTest.java
deleted file mode 100644
index c14dff9..0000000
--- a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/FileHolderImplUnitTest.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.carbon.datastorage.filesystem.store.impl;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-
-import org.apache.carbondata.core.datastorage.store.impl.FileHolderImpl;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThat;
-import static org.pentaho.di.core.util.Assert.assertNull;
-
-public class FileHolderImplUnitTest {
-
- private static FileHolderImpl fileHolder;
- private static FileHolderImpl fileHolderWithCapacity;
- private static String fileName;
- private static String fileNameWithEmptyContent;
- private static File file;
- private static File fileWithEmptyContent;
-
- @BeforeClass public static void setup() {
- fileHolder = new FileHolderImpl();
- fileHolderWithCapacity = new FileHolderImpl(50);
- file = new File("Test.carbondata");
- fileWithEmptyContent = new File("TestEXception.carbondata");
-
- if (!file.exists()) try {
- file.createNewFile();
- } catch (IOException e) {
- e.printStackTrace();
- }
- if (!fileWithEmptyContent.exists()) try {
- fileWithEmptyContent.createNewFile();
- } catch (IOException e) {
- e.printStackTrace();
- }
- try {
- FileOutputStream of = new FileOutputStream(file, true);
- BufferedWriter br = new BufferedWriter(new OutputStreamWriter(of, "UTF-8"));
- br.write("Hello World");
- br.close();
- } catch (Exception e) {
- e.getMessage();
- }
- fileName = file.getAbsolutePath();
- fileNameWithEmptyContent = fileWithEmptyContent.getAbsolutePath();
- }
-
- @AfterClass public static void tearDown() throws IOException {
- file.delete();
- fileWithEmptyContent.delete();
- fileHolder.finish();
- }
-
- @Test public void testReadByteArray() throws IOException {
- byte[] result = fileHolder.readByteArray(fileName, 1);
- byte[] expected_result = new byte[] { 72 };
- assertThat(result, is(equalTo(expected_result)));
- }
-
- @Test public void testReadByteArrayWithFilePath() throws IOException {
- byte[] result = fileHolder.readByteArray(fileName, 2L, 2);
- byte[] expected_result = { 108, 108 };
- assertThat(result, is(equalTo(expected_result)));
- }
-
- @Test public void testReadLong() throws IOException {
- long actualResult = fileHolder.readLong(fileName, 1L);
- long expectedResult = 7308335519855243122L;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test public void testReadLongForIoException() throws IOException {
- fileHolder.readLong(fileNameWithEmptyContent, 1L);
- }
-
- @Test public void testReadIntForIoException() throws IOException {
- fileHolder.readInt(fileNameWithEmptyContent, 1L);
- }
-
- @Test public void testReadInt() throws IOException {
- int actualResult = fileHolder.readInt(fileName, 1L);
- int expectedResult = 1701604463;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test public void testReadIntWithFileName() throws IOException {
- int actualResult = fileHolder.readInt(fileName);
- int expectedResult = 1701604463;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test public void testReadIntWithFileNameForIOException() throws IOException {
- fileHolder.readInt(fileNameWithEmptyContent);
-
- }
-
- @Test public void testDouble() throws IOException {
- double actualResult = fileHolder.readDouble(fileName, 1L);
- double expectedResult = 7.3083355198552433E18;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test public void testDoubleForIoException() throws IOException {
- fileHolder.readDouble(fileNameWithEmptyContent, 1L);
-
- }
-
- @Test public void testDoubleForIoExceptionwithUpdateCache() throws Exception {
- new MockUp<FileSystem>() {
- @SuppressWarnings("unused") @Mock public FSDataInputStream open(Path file)
- throws IOException {
- throw new IOException();
- }
-
- };
- try {
- fileHolder.readDouble(fileName, 1L);
- } catch (Exception e) {
- assertNull(e.getMessage());
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorderImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorderImplTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorderImplTest.java
deleted file mode 100644
index 4322817..0000000
--- a/core/src/test/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorderImplTest.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.carbon.querystatistics;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.core.stats.DriverQueryStatisticsRecorderImpl;
-import org.apache.carbondata.core.stats.QueryStatistic;
-import org.apache.carbondata.core.stats.QueryStatisticsConstants;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static junit.framework.TestCase.assertNotNull;
-import static org.pentaho.di.core.util.Assert.assertNull;
-
-public class DriverQueryStatisticsRecorderImplTest {
- private static DriverQueryStatisticsRecorderImpl driverQueryStatisticsRecorderImpl = null;
- private static QueryStatistic queryStasticsWithSQL_PARSE = null;
- private static QueryStatistic queryStasticsWithLoadMetaData = null;
- private static QueryStatistic queryStasticsWithLOAD_BLOCKS_DRIVER = null;
- private static QueryStatistic queryStasticsWithBLOCK_ALLOCATION = null;
- private static QueryStatistic queryStasticsWithBLOCK_IDENTIFICATION = null;
-
- private static List<QueryStatistic> queryStatisticList;
-
- @BeforeClass public static void setUp() {
- driverQueryStatisticsRecorderImpl = DriverQueryStatisticsRecorderImpl.getInstance();
- queryStasticsWithSQL_PARSE = new QueryStatistic();
- queryStasticsWithLoadMetaData = new QueryStatistic();
- queryStasticsWithLOAD_BLOCKS_DRIVER = new QueryStatistic();
- queryStasticsWithBLOCK_ALLOCATION = new QueryStatistic();
- queryStasticsWithBLOCK_IDENTIFICATION = new QueryStatistic();
- queryStatisticList = new ArrayList<>();
-
- }
-
- @Test public void testCollectDriverStatistics() {
- queryStasticsWithLOAD_BLOCKS_DRIVER
- .addCountStatistic(QueryStatisticsConstants.LOAD_BLOCKS_DRIVER, 3);
- queryStasticsWithLOAD_BLOCKS_DRIVER
- .addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_DRIVER, 5);
- queryStasticsWithLOAD_BLOCKS_DRIVER
- .addFixedTimeStatistic(QueryStatisticsConstants.LOAD_BLOCKS_DRIVER, 5);
- queryStatisticList.add(queryStasticsWithLOAD_BLOCKS_DRIVER);
- queryStasticsWithBLOCK_ALLOCATION
- .addCountStatistic(QueryStatisticsConstants.BLOCK_ALLOCATION, 3);
- queryStasticsWithBLOCK_ALLOCATION.addStatistics(QueryStatisticsConstants.BLOCK_ALLOCATION, 5);
- queryStasticsWithLOAD_BLOCKS_DRIVER
- .addFixedTimeStatistic(QueryStatisticsConstants.LOAD_BLOCKS_DRIVER, 5);
- queryStatisticList.add(queryStasticsWithBLOCK_ALLOCATION);
- queryStasticsWithBLOCK_IDENTIFICATION
- .addCountStatistic(QueryStatisticsConstants.BLOCK_IDENTIFICATION, 3);
- queryStasticsWithBLOCK_IDENTIFICATION
- .addStatistics(QueryStatisticsConstants.BLOCK_IDENTIFICATION, 5);
- queryStatisticList.add(queryStasticsWithBLOCK_IDENTIFICATION);
- queryStasticsWithLoadMetaData.addCountStatistic(QueryStatisticsConstants.LOAD_META, 3);
- queryStasticsWithLoadMetaData.addStatistics(QueryStatisticsConstants.LOAD_META, 5);
- queryStatisticList.add(queryStasticsWithLoadMetaData);
- queryStasticsWithSQL_PARSE.addCountStatistic(QueryStatisticsConstants.SQL_PARSE, 3);
- queryStasticsWithSQL_PARSE.addStatistics(QueryStatisticsConstants.SQL_PARSE, 5);
- queryStatisticList.add(queryStasticsWithSQL_PARSE);
-
- String result =
- driverQueryStatisticsRecorderImpl.collectDriverStatistics(queryStatisticList, "query1");
- assertNotNull(result);
- }
-
- @Test public void testCollectDriverStatisticsWithBlock_Allocation() {
- queryStatisticList.clear();
- queryStasticsWithBLOCK_ALLOCATION
- .addCountStatistic(QueryStatisticsConstants.BLOCK_ALLOCATION, 3);
- queryStasticsWithBLOCK_ALLOCATION.addStatistics(QueryStatisticsConstants.BLOCK_ALLOCATION, 5);
- queryStatisticList.add(queryStasticsWithBLOCK_ALLOCATION);
- String result =
- driverQueryStatisticsRecorderImpl.collectDriverStatistics(queryStatisticList, "query1");
- assertNull(result);
- }
-
- @Test public void testCollectDriverStatisticsWithBlock_AllocationAndBlock_Identification() {
- queryStatisticList.clear();
- queryStasticsWithBLOCK_ALLOCATION
- .addCountStatistic(QueryStatisticsConstants.BLOCK_ALLOCATION, 3);
- queryStasticsWithBLOCK_ALLOCATION.addStatistics(QueryStatisticsConstants.BLOCK_ALLOCATION, 5);
- queryStatisticList.add(queryStasticsWithBLOCK_ALLOCATION);
- queryStasticsWithBLOCK_IDENTIFICATION
- .addCountStatistic(QueryStatisticsConstants.BLOCK_IDENTIFICATION, 3);
- queryStasticsWithBLOCK_IDENTIFICATION
- .addStatistics(QueryStatisticsConstants.BLOCK_IDENTIFICATION, 5);
- queryStatisticList.add(queryStasticsWithBLOCK_IDENTIFICATION);
- String result =
- driverQueryStatisticsRecorderImpl.collectDriverStatistics(queryStatisticList, "query1");
- assertNotNull(result);
- }
-
- @Test
- public void testCollectDriverStatisticsWithBlock_AllocationAndBlock_IdentificationForException() {
- queryStatisticList.clear();
-
- driverQueryStatisticsRecorderImpl = DriverQueryStatisticsRecorderImpl.getInstance();
-
- String result =
- driverQueryStatisticsRecorderImpl.collectDriverStatistics(queryStatisticList, "query1");
- assertNull(result);
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/carbon/querystatistics/QueryStasticsRecorderImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/querystatistics/QueryStasticsRecorderImplTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/querystatistics/QueryStasticsRecorderImplTest.java
deleted file mode 100644
index fe2f1ea..0000000
--- a/core/src/test/java/org/apache/carbondata/core/carbon/querystatistics/QueryStasticsRecorderImplTest.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.carbon.querystatistics;
-
-import org.apache.carbondata.core.stats.QueryStatistic;
-import org.apache.carbondata.core.stats.QueryStatisticsConstants;
-import org.apache.carbondata.core.stats.QueryStatisticsRecorderImpl;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static junit.framework.TestCase.assertNotNull;
-
-public class QueryStasticsRecorderImplTest {
-
- private static QueryStatisticsRecorderImpl queryStasticsRecorderImpl = null;
- private static QueryStatistic queryStatistic = null;
- private static QueryStatistic queryStatisticWithLOAD_BLOCKS_EXECUTOR = null;
- private static QueryStatistic queryStatisticWithSCAN_BLOCKS_TIME = null;
- private static QueryStatistic queryStatisticWithSCAN_BLOCKS_NUM = null;
- private static QueryStatistic queryStatisticWithLOAD_DICTIONARY = null;
- private static QueryStatistic queryStatisticWithRESULT_SIZE = null;
- private static QueryStatistic queryStatisticWithEXECUTOR_PART = null;
- private static QueryStatistic queryStatisticWithTOTAL_BLOCKLET_NUM = null;
- private static QueryStatistic queryStatisticWithVALID_SCAN_BLOCKLET_NUM = null;
-
- @BeforeClass public static void setUp() {
- queryStatisticWithLOAD_BLOCKS_EXECUTOR = new QueryStatistic();
- queryStasticsRecorderImpl = new QueryStatisticsRecorderImpl(System.nanoTime() + "");
- queryStasticsRecorderImpl.logStatisticsAsTableDriver();
- queryStatisticWithLOAD_BLOCKS_EXECUTOR
- .addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_EXECUTOR, 5L);
- queryStatisticWithLOAD_BLOCKS_EXECUTOR
- .addCountStatistic(QueryStatisticsConstants.LOAD_BLOCKS_EXECUTOR, 5L);
- queryStatisticWithLOAD_BLOCKS_EXECUTOR
- .addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_EXECUTOR, 5L);
- queryStasticsRecorderImpl.logStatistics();
- queryStasticsRecorderImpl.recordStatistics(queryStatisticWithLOAD_BLOCKS_EXECUTOR);
- queryStatisticWithSCAN_BLOCKS_TIME = new QueryStatistic();
- queryStatisticWithSCAN_BLOCKS_TIME.addStatistics(QueryStatisticsConstants.SCAN_BLOCKS_TIME, 5L);
- queryStatisticWithSCAN_BLOCKS_TIME
- .addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKS_TIME, 5L);
- queryStatisticWithSCAN_BLOCKS_TIME.addStatistics(QueryStatisticsConstants.SCAN_BLOCKS_TIME, 5L);
- queryStasticsRecorderImpl.logStatistics();
- queryStasticsRecorderImpl.recordStatistics(queryStatisticWithSCAN_BLOCKS_TIME);
- queryStatisticWithSCAN_BLOCKS_NUM = new QueryStatistic();
- queryStatisticWithSCAN_BLOCKS_NUM.addStatistics(QueryStatisticsConstants.SCAN_BLOCKS_NUM, 5L);
- queryStatisticWithSCAN_BLOCKS_NUM
- .addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKS_NUM, 5L);
- queryStatisticWithSCAN_BLOCKS_NUM.addStatistics(QueryStatisticsConstants.SCAN_BLOCKS_NUM, 5L);
- queryStasticsRecorderImpl.logStatistics();
- queryStasticsRecorderImpl.recordStatistics(queryStatisticWithSCAN_BLOCKS_NUM);
- queryStatisticWithLOAD_DICTIONARY = new QueryStatistic();
- queryStatisticWithLOAD_DICTIONARY.addStatistics(QueryStatisticsConstants.LOAD_DICTIONARY, 5L);
- queryStatisticWithLOAD_DICTIONARY
- .addCountStatistic(QueryStatisticsConstants.LOAD_DICTIONARY, 5L);
- queryStatisticWithLOAD_DICTIONARY.addStatistics(QueryStatisticsConstants.LOAD_DICTIONARY, 5L);
- queryStasticsRecorderImpl.logStatistics();
- queryStasticsRecorderImpl.recordStatistics(queryStatisticWithLOAD_DICTIONARY);
- queryStatisticWithRESULT_SIZE = new QueryStatistic();
- queryStatisticWithRESULT_SIZE.addStatistics(QueryStatisticsConstants.RESULT_SIZE, 5L);
- queryStatisticWithRESULT_SIZE.addCountStatistic(QueryStatisticsConstants.RESULT_SIZE, 5L);
- queryStatisticWithRESULT_SIZE.addStatistics(QueryStatisticsConstants.RESULT_SIZE, 5L);
- queryStasticsRecorderImpl.logStatistics();
- queryStasticsRecorderImpl.recordStatistics(queryStatisticWithRESULT_SIZE);
- queryStatisticWithEXECUTOR_PART = new QueryStatistic();
- queryStatisticWithEXECUTOR_PART.addStatistics(QueryStatisticsConstants.EXECUTOR_PART, 5L);
- queryStatisticWithEXECUTOR_PART.addCountStatistic(QueryStatisticsConstants.EXECUTOR_PART, 5L);
- queryStatisticWithEXECUTOR_PART.addStatistics(QueryStatisticsConstants.EXECUTOR_PART, 5L);
- queryStasticsRecorderImpl.logStatistics();
- queryStasticsRecorderImpl.recordStatistics(queryStatisticWithEXECUTOR_PART);
- queryStatisticWithTOTAL_BLOCKLET_NUM = new QueryStatistic();
- queryStatisticWithTOTAL_BLOCKLET_NUM
- .addStatistics(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM, 5L);
- queryStatisticWithTOTAL_BLOCKLET_NUM
- .addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM, 5L);
- queryStatisticWithTOTAL_BLOCKLET_NUM
- .addStatistics(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM, 5L);
- queryStasticsRecorderImpl.logStatistics();
- queryStasticsRecorderImpl.recordStatistics(queryStatisticWithTOTAL_BLOCKLET_NUM);
- queryStatisticWithVALID_SCAN_BLOCKLET_NUM = new QueryStatistic();
- queryStatisticWithVALID_SCAN_BLOCKLET_NUM
- .addStatistics(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM, 5L);
- queryStatisticWithVALID_SCAN_BLOCKLET_NUM
- .addCountStatistic(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM, 5L);
- queryStatisticWithVALID_SCAN_BLOCKLET_NUM
- .addStatistics(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM, 5L);
- queryStasticsRecorderImpl.logStatistics();
- queryStasticsRecorderImpl.recordStatistics(queryStatisticWithVALID_SCAN_BLOCKLET_NUM);
- queryStasticsRecorderImpl.logStatisticsAsTableExecutor();
- }
-
- @Test public void testcollectExecutorStatistics() {
- assertNotNull(queryStasticsRecorderImpl.collectExecutorStatistics());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/datastorage/CompressedDataMeasureWrapperTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/CompressedDataMeasureWrapperTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/CompressedDataMeasureWrapperTest.java
new file mode 100644
index 0000000..8bc6af4
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/datastorage/CompressedDataMeasureWrapperTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage;
+
+import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.datastorage.impl.CompressedDataMeasureDataWrapper;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertEquals;
+
+public class CompressedDataMeasureWrapperTest {
+ private static CompressedDataMeasureDataWrapper compressedDataMeasureDataWrapper;
+ private static CarbonReadDataHolder carbonReadDataHolder;
+ private static CarbonReadDataHolder[] carbonReadDataHolders;
+
+ @BeforeClass public static void setUp() {
+ carbonReadDataHolder = new CarbonReadDataHolder(null);
+ carbonReadDataHolders = new CarbonReadDataHolder[] { carbonReadDataHolder };
+
+ }
+
+ @Test public void testGetValues() {
+ compressedDataMeasureDataWrapper = new CompressedDataMeasureDataWrapper(carbonReadDataHolders);
+ int expected = 1;
+ int actual = compressedDataMeasureDataWrapper.getValues().length;
+ assertEquals(expected, actual);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/datastorage/DFSFileHolderImplUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/DFSFileHolderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/DFSFileHolderImplUnitTest.java
new file mode 100644
index 0000000..62700db
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/datastorage/DFSFileHolderImplUnitTest.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.datastorage;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+
+import org.apache.carbondata.core.datastorage.impl.DFSFileHolderImpl;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.pentaho.di.core.util.Assert.assertNull;
+
+public class DFSFileHolderImplUnitTest {
+
+ private static DFSFileHolderImpl dfsFileHolder;
+ private static String fileName;
+ private static String fileNameWithEmptyContent;
+ private static File file;
+ private static File fileWithEmptyContent;
+
+ @BeforeClass public static void setup() {
+ dfsFileHolder = new DFSFileHolderImpl();
+ file = new File("Test.carbondata");
+ fileWithEmptyContent = new File("TestEXception.carbondata");
+
+ if (!file.exists()) try {
+ file.createNewFile();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ if (!fileWithEmptyContent.exists()) try {
+ fileWithEmptyContent.createNewFile();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
+ FileOutputStream of = new FileOutputStream(file, true);
+ BufferedWriter br = new BufferedWriter(new OutputStreamWriter(of, "UTF-8"));
+ br.write("Hello World");
+ br.close();
+ } catch (Exception e) {
+ e.getMessage();
+ }
+ fileName = file.getAbsolutePath();
+ fileNameWithEmptyContent = fileWithEmptyContent.getAbsolutePath();
+ }
+
+ @AfterClass public static void tearDown() throws IOException {
+ file.delete();
+ fileWithEmptyContent.delete();
+ dfsFileHolder.finish();
+ }
+
+ @Test public void testReadByteArray() throws IOException {
+ byte[] result = dfsFileHolder.readByteArray(fileName, 1);
+ byte[] expected_result = new byte[] { 72 };
+ assertThat(result, is(equalTo(expected_result)));
+ }
+
+ @Test public void testReadByteArrayWithFilePath() throws IOException {
+ byte[] result = dfsFileHolder.readByteArray(fileName, 2L, 2);
+ byte[] expected_result = { 108, 108 };
+ assertThat(result, is(equalTo(expected_result)));
+ }
+
+ @Test public void testReadLong() throws IOException {
+ long actualResult = dfsFileHolder.readLong(fileName, 1L);
+ long expectedResult = 7308335519855243122L;
+ assertThat(actualResult, is(equalTo(expectedResult)));
+ }
+
+ @Test(expected = IOException.class)
+ public void testReadLongForIoException() throws IOException {
+ dfsFileHolder.readLong(fileNameWithEmptyContent, 1L);
+ }
+
+ @Test(expected = IOException.class)
+ public void testReadIntForIoException() throws IOException{
+ dfsFileHolder.readInt(fileNameWithEmptyContent, 1L);
+ }
+
+ @Test public void testReadInt() throws IOException {
+ int actualResult = dfsFileHolder.readInt(fileName, 1L);
+ int expectedResult = 1701604463;
+ assertThat(actualResult, is(equalTo(expectedResult)));
+ }
+
+ @Test public void testReadIntWithFileName() throws IOException {
+ int actualResult = dfsFileHolder.readInt(fileName);
+ int expectedResult = 1701604463;
+ assertThat(actualResult, is(equalTo(expectedResult)));
+ }
+
+ @Test(expected = IOException.class)
+ public void testReadIntWithFileNameForIOException() throws IOException {
+ dfsFileHolder.readInt(fileNameWithEmptyContent);
+ }
+
+ @Test public void testDouble() throws IOException {
+ double actualResult = dfsFileHolder.readDouble(fileName, 1L);
+ double expectedResult = 7.3083355198552433E18;
+ assertThat(actualResult, is(equalTo(expectedResult)));
+ }
+
+ @Test(expected = IOException.class)
+ public void testDoubleForIoException() throws IOException {
+ dfsFileHolder.readDouble(fileNameWithEmptyContent, 1L);
+ }
+
+ @Test
+ public void testDoubleForIoExceptionwithUpdateCache() throws IOException {
+ new MockUp<FileSystem>() {
+ @SuppressWarnings("unused") @Mock public FSDataInputStream open(Path file)
+ throws IOException {
+ throw new IOException();
+ }
+
+ };
+ dfsFileHolder.readDouble(fileName, 1L);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/datastorage/FileFactoryImplUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/FileFactoryImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/FileFactoryImplUnitTest.java
new file mode 100644
index 0000000..51874c0
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/datastorage/FileFactoryImplUnitTest.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertNotNull;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class FileFactoryImplUnitTest {
+
+ private static String filePath;
+
+ @AfterClass
+ public static void tearDown() {
+ File file = new File(filePath);
+ if (file.exists()) {
+ file.delete();
+ }
+
+ File file1 = new File(".TestFileFactory.carbondata.crc");
+ if (file1.exists()) {
+ file1.delete();
+ }
+ }
+
+ @BeforeClass public static void setUp() {
+ filePath = "TestFileFactory.carbondata";
+ }
+
+ @Test public void testFileExistsForVIEWFSType() throws IOException {
+ FileFactory.isFileExist("fakefilePath", FileFactory.FileType.VIEWFS);
+ }
+
+ @Test public void testFileExistsForDefaultType() throws IOException {
+ FileFactory.isFileExist("fakefilePath", FileFactory.FileType.LOCAL);
+ }
+
+ @Test public void testFileExistsForDefaultTypeWithPerformFileCheck() throws IOException {
+ assertTrue(FileFactory.isFileExist(filePath, FileFactory.FileType.LOCAL, true));
+ }
+
+ @Test public void testFileExistsForDefaultTypeWithOutPerformFileCheck() throws IOException {
+ assertFalse(FileFactory.isFileExist("fakefilePath", FileFactory.FileType.LOCAL, false));
+ }
+
+ @Test public void testFileExistsForVIEWFSTypeWithPerformFileCheck() throws IOException {
+ assertTrue(FileFactory.isFileExist(filePath, FileFactory.FileType.VIEWFS, true));
+ }
+
+ @Test public void testFileExistsForVIEWFSTypeWithOutPerformFileCheck() throws IOException {
+ assertFalse(FileFactory.isFileExist("fakefilePath", FileFactory.FileType.VIEWFS, false));
+ }
+
+ @Test public void testCreateNewFileWithDefaultFileType() throws IOException {
+ tearDown();
+ assertTrue(FileFactory.createNewFile(filePath, FileFactory.FileType.LOCAL));
+ }
+
+ @Test public void testCreateNewLockFileWithDefaultFileType() throws IOException {
+ tearDown();
+ assertTrue(FileFactory.createNewLockFile(filePath, FileFactory.FileType.LOCAL));
+ }
+
+ @Test public void testCreateNewLockFileWithViewFsFileType() throws IOException {
+ tearDown();
+ assertTrue(FileFactory.createNewLockFile(filePath, FileFactory.FileType.VIEWFS));
+ }
+
+ @Test public void testCreateNewLockFileWithViewFsFileTypeWhenFileExists() throws IOException {
+ assertFalse(FileFactory.createNewLockFile(filePath, FileFactory.FileType.VIEWFS));
+ }
+
+ @Test public void testCreateNewFileWithDefaultFileTypeWhenFileExists() throws IOException {
+ assertFalse(FileFactory.createNewFile(filePath, FileFactory.FileType.LOCAL));
+ }
+
+ @Test public void testCreateNewFileWithVIEWFSFileType() throws IOException {
+ File file = new File(filePath);
+ if (file.exists()) {
+ file.delete();
+ }
+ assertTrue(FileFactory.createNewFile(filePath, FileFactory.FileType.VIEWFS));
+ }
+
+ @Test public void testCreateNewFileWithVIEWFSFileTypeWhenFileExists() throws IOException {
+ assertFalse(FileFactory.createNewFile(filePath, FileFactory.FileType.VIEWFS));
+ }
+
+ @Test public void testMkDirWithVIEWFSFileType() throws IOException {
+ new MockUp<FileSystem>() {
+ @SuppressWarnings("unused") @Mock public boolean mkdirs(Path file) throws IOException {
+ {
+ return true;
+ }
+ }
+ };
+ tearDown();
+ assertTrue(FileFactory.mkdirs(filePath, FileFactory.FileType.VIEWFS));
+ }
+
+ @Test public void testGetDataOutputStreamUsingAppendeForException() {
+ try {
+ FileFactory.getDataOutputStreamUsingAppend(filePath, FileFactory.FileType.VIEWFS);
+ } catch (Exception exception) {
+ assertEquals("Not supported", exception.getMessage());
+ }
+ }
+
+ @Test public void getDataOutputStreamForVIEWFSType() throws IOException {
+ assertNotNull(FileFactory.getDataOutputStream(filePath, FileFactory.FileType.VIEWFS));
+ }
+
+ @Test public void getDataOutputStreamForLocalType() throws IOException {
+ assertNotNull(FileFactory.getDataOutputStream(filePath, FileFactory.FileType.LOCAL));
+ }
+
+ @Test public void testGetCarbonFile() throws IOException {
+ FileFactory.getDataOutputStream(filePath, FileFactory.FileType.VIEWFS);
+ assertNotNull(FileFactory.getCarbonFile(filePath, FileFactory.FileType.HDFS));
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/datastorage/FileHolderImplUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/FileHolderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/FileHolderImplUnitTest.java
new file mode 100644
index 0000000..854b14d
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/datastorage/FileHolderImplUnitTest.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.datastorage;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+
+import org.apache.carbondata.core.datastorage.impl.FileHolderImpl;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.pentaho.di.core.util.Assert.assertNull;
+
+public class FileHolderImplUnitTest {
+
+ private static FileHolderImpl fileHolder;
+ private static FileHolderImpl fileHolderWithCapacity;
+ private static String fileName;
+ private static String fileNameWithEmptyContent;
+ private static File file;
+ private static File fileWithEmptyContent;
+
+ @BeforeClass public static void setup() {
+ fileHolder = new FileHolderImpl();
+ fileHolderWithCapacity = new FileHolderImpl(50);
+ file = new File("Test.carbondata");
+ fileWithEmptyContent = new File("TestEXception.carbondata");
+
+ if (!file.exists()) try {
+ file.createNewFile();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ if (!fileWithEmptyContent.exists()) try {
+ fileWithEmptyContent.createNewFile();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
+ FileOutputStream of = new FileOutputStream(file, true);
+ BufferedWriter br = new BufferedWriter(new OutputStreamWriter(of, "UTF-8"));
+ br.write("Hello World");
+ br.close();
+ } catch (Exception e) {
+ e.getMessage();
+ }
+ fileName = file.getAbsolutePath();
+ fileNameWithEmptyContent = fileWithEmptyContent.getAbsolutePath();
+ }
+
+ @AfterClass public static void tearDown() throws IOException {
+ file.delete();
+ fileWithEmptyContent.delete();
+ fileHolder.finish();
+ }
+
+ @Test public void testReadByteArray() throws IOException {
+ byte[] result = fileHolder.readByteArray(fileName, 1);
+ byte[] expected_result = new byte[] { 72 };
+ assertThat(result, is(equalTo(expected_result)));
+ }
+
+ @Test public void testReadByteArrayWithFilePath() throws IOException {
+ byte[] result = fileHolder.readByteArray(fileName, 2L, 2);
+ byte[] expected_result = { 108, 108 };
+ assertThat(result, is(equalTo(expected_result)));
+ }
+
+ @Test public void testReadLong() throws IOException {
+ long actualResult = fileHolder.readLong(fileName, 1L);
+ long expectedResult = 7308335519855243122L;
+ assertThat(actualResult, is(equalTo(expectedResult)));
+ }
+
+ @Test public void testReadLongForIoException() throws IOException {
+ fileHolder.readLong(fileNameWithEmptyContent, 1L);
+ }
+
+ @Test public void testReadIntForIoException() throws IOException {
+ fileHolder.readInt(fileNameWithEmptyContent, 1L);
+ }
+
+ @Test public void testReadInt() throws IOException {
+ int actualResult = fileHolder.readInt(fileName, 1L);
+ int expectedResult = 1701604463;
+ assertThat(actualResult, is(equalTo(expectedResult)));
+ }
+
+ @Test public void testReadIntWithFileName() throws IOException {
+ int actualResult = fileHolder.readInt(fileName);
+ int expectedResult = 1701604463;
+ assertThat(actualResult, is(equalTo(expectedResult)));
+ }
+
+ @Test public void testReadIntWithFileNameForIOException() throws IOException {
+ fileHolder.readInt(fileNameWithEmptyContent);
+
+ }
+
+ @Test public void testDouble() throws IOException {
+ double actualResult = fileHolder.readDouble(fileName, 1L);
+ double expectedResult = 7.3083355198552433E18;
+ assertThat(actualResult, is(equalTo(expectedResult)));
+ }
+
+ @Test public void testDoubleForIoException() throws IOException {
+ fileHolder.readDouble(fileNameWithEmptyContent, 1L);
+
+ }
+
+ @Test public void testDoubleForIoExceptionwithUpdateCache() throws Exception {
+ new MockUp<FileSystem>() {
+ @SuppressWarnings("unused") @Mock public FSDataInputStream open(Path file)
+ throws IOException {
+ throw new IOException();
+ }
+
+ };
+ try {
+ fileHolder.readDouble(fileName, 1L);
+ } catch (Exception e) {
+ assertNull(e.getMessage());
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreDataHolderUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreDataHolderUnitTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreDataHolderUnitTest.java
new file mode 100644
index 0000000..29d2eae
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreDataHolderUnitTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.datastorage.columnar;
+
+import java.util.List;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class ColumnarKeyStoreDataHolderUnitTest {
+
+ private static ColumnarKeyStoreDataHolder columnarKeyStoreDataHolder;
+ private static ColumnarKeyStoreMetadata columnarKeyStoreMetadata;
+
+ @BeforeClass public static void setup() {
+ byte[] keyBlockData = new byte[] { 16, 8, 32, 40, 8, 8, 8 };
+ int eachRowSize = 2;
+ int[] reverseIndex = new int[] { 1, 5, 6, 3, 8 };
+ columnarKeyStoreMetadata = new ColumnarKeyStoreMetadata(eachRowSize);
+ columnarKeyStoreMetadata.setColumnReverseIndex(reverseIndex);
+ columnarKeyStoreDataHolder =
+ new ColumnarKeyStoreDataHolder(keyBlockData, columnarKeyStoreMetadata);
+ }
+
+ @Test public void testGetSurrogateKeyWithNullINGetColumnReverseIndex() {
+ byte[] keyBlockData = new byte[] { 16, 8, 32, 40, 8, 8, 8 };
+ int eachRowSize = 1;
+ ColumnarKeyStoreMetadata columnarKeyStoreMetadata = new ColumnarKeyStoreMetadata(eachRowSize);
+ ColumnarKeyStoreDataHolder columnarKeyStoreDataHolderNew =
+ new ColumnarKeyStoreDataHolder(keyBlockData, columnarKeyStoreMetadata);
+ int columnIndex = 5;
+ int expected_result = 8;
+ int result = columnarKeyStoreDataHolderNew.getSurrogateKey(columnIndex);
+ assertEquals(expected_result, result);
+ }
+
+ @Test public void testGetSurrogateKeyWithNullINGetColumnReverseIndexAndRowSizeTwo() {
+ byte[] keyBlockData = new byte[] { 16, 8, 32, 40, 8, 8, 8 };
+ int eachRowSize = 2;
+ ColumnarKeyStoreMetadata columnarKeyStoreMetadata = new ColumnarKeyStoreMetadata(eachRowSize);
+ ColumnarKeyStoreDataHolder columnarKeyStoreDataHolderNew =
+ new ColumnarKeyStoreDataHolder(keyBlockData, columnarKeyStoreMetadata);
+ int columnIndex = 0;
+ int expected_result = 4104;
+ int result = columnarKeyStoreDataHolderNew.getSurrogateKey(columnIndex);
+ assertEquals(expected_result, result);
+ }
+
+ @Test public void testGetSurrogateKeyWithNotNullINGetColumnReverseIndex() {
+ int columnIndex = 0;
+ int expected_result = 8232;
+ int result = columnarKeyStoreDataHolder.getSurrogateKey(columnIndex);
+ assertEquals(expected_result, result);
+ }
+
+ @Test(expected = ArrayIndexOutOfBoundsException.class)
+ public void testExceptionInGetSurrogateKey() {
+ int columnIndex = 10;
+ int expected_result = 8232;
+ int result = columnarKeyStoreDataHolder.getSurrogateKey(columnIndex);
+ assertEquals(expected_result, result);
+ }
+
+ @Test public void testGetSurrogateKeyWithListOfByteWhileCreatingObject() {
+ byte[] keyBlockData = new byte[] { 32, 64, 32, 40, 64, 8, 8 };
+ List<byte[]> noDictionaryValBasedKeyBlockData = new java.util.ArrayList<>();
+ noDictionaryValBasedKeyBlockData.add(keyBlockData);
+ new ColumnarKeyStoreDataHolder(columnarKeyStoreMetadata);
+ int columnIndex = 0;
+ int expected_result = 8232;
+ int result = columnarKeyStoreDataHolder.getSurrogateKey(columnIndex);
+ assertEquals(expected_result, result);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/AlluxioCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/AlluxioCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/AlluxioCarbonFileTest.java
new file mode 100644
index 0000000..49323d8
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/AlluxioCarbonFileTest.java
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.filesystem;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Options;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class AlluxioCarbonFileTest {
+
+ private static AlluxioCarbonFile alluxioCarbonFile;
+ private static FileStatus fileStatus;
+ private static FileStatus fileStatusWithOutDirectoryPermission;
+ private static String fileName;
+ private static File file;
+
+
+ @BeforeClass
+ static public void setUp() {
+ file = new File("Test.carbondata");
+ if (!file.exists())
+ try {
+ file.createNewFile();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
+ FileOutputStream oFile = new FileOutputStream(file, true);
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
+
+ fileStatus = new FileStatus(12L, true, 60, 120l, 180L, new Path(file.getAbsolutePath()));
+ fileStatusWithOutDirectoryPermission = new FileStatus(12L, false, 60, 120l, 180L, new Path(file.getAbsolutePath()));
+ fileName = file.getAbsolutePath();
+ alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
+ }
+
+ @AfterClass
+ static public void cleanUp() {
+ file.delete();
+ }
+
+ @Test
+ public void testRenameForceForException() throws IOException {
+
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ throw new IOException();
+ }
+
+ };
+ alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
+ alluxioCarbonFile.renameForce(fileName);
+ }
+
+ @Test
+ public void testListFilesWithOutDirectoryPermission() {
+ alluxioCarbonFile = new AlluxioCarbonFile(fileStatusWithOutDirectoryPermission);
+ assertTrue(alluxioCarbonFile.listFiles() == null);
+ }
+
+ @Test
+ public void testConstructorWithFilePath() {
+ alluxioCarbonFile = new AlluxioCarbonFile(file.getAbsolutePath());
+ assertTrue(alluxioCarbonFile instanceof AlluxioCarbonFile);
+ }
+
+ @Test
+ public void testListFilesForNullListStatus() {
+ alluxioCarbonFile = new AlluxioCarbonFile(fileStatusWithOutDirectoryPermission);
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public FileStatus[] listStatus(Path var1) throws IOException {
+
+ return null;
+ }
+
+ };
+ alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
+ assertTrue(alluxioCarbonFile.listFiles().length == 0);
+ }
+
+ @Test
+ public void testListDirectory() {
+ alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public FileStatus[] listStatus(Path var1) throws IOException {
+
+ FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
+ return fileStatus;
+ }
+
+ };
+
+ assertTrue(alluxioCarbonFile.listFiles().length == 1);
+ }
+
+ @Test
+ public void testListFilesForException() throws IOException {
+ alluxioCarbonFile = new AlluxioCarbonFile(fileStatusWithOutDirectoryPermission);
+
+ new MockUp<FileStatus>() {
+ @Mock
+ public Path getPath() {
+ return new Path(file.getAbsolutePath());
+ }
+
+ };
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ throw new IOException();
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public FileStatus[] listStatus(Path var1) throws IOException {
+
+ throw new IOException();
+ }
+
+ };
+ alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
+ alluxioCarbonFile.listFiles();
+ }
+
+ @Test
+ public void testListFilesWithCarbonFilter() {
+ CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
+
+ @Override
+ public boolean accept(CarbonFile file) {
+ return true;
+ }
+ };
+ alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
+ assertTrue(alluxioCarbonFile.listFiles(carbonFileFilter).length == 1);
+ }
+
+ @Test
+ public void testlistFilesWithoutFilter() {
+ CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
+
+ @Override
+ public boolean accept(CarbonFile file) {
+ return false;
+ }
+ };
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public FileStatus[] listStatus(Path var1) throws IOException {
+
+ FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
+ return fileStatus;
+ }
+
+ };
+ alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
+ assertTrue(alluxioCarbonFile.listFiles(carbonFileFilter).length == 0);
+ }
+
+ @Test
+ public void testGetParentFile() {
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+ new MockUp<Path>() {
+ @Mock
+ public Path getParent() {
+ return new Path(file.getAbsolutePath()
+ );
+ }
+
+ };
+ new MockUp<FileStatus>() {
+ @Mock
+ public Path getPath() {
+ return new Path(file.getAbsolutePath());
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public FileStatus getFileStatus(Path path) throws IOException {
+
+ return new FileStatus(12L, true, 60, 120l, 180L, new Path(file.getAbsolutePath()));
+ }
+
+ };
+
+ alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
+ assertFalse(alluxioCarbonFile.getParentFile().equals(null));
+ }
+
+ @Test
+ public void testForNonDisributedSystem() {
+ alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new WebHdfsFileSystem();
+ }
+
+ };
+ assertFalse(alluxioCarbonFile.renameForce(fileName));
+ }
+
+ @Test
+ public void testrenameForceForDisributedSystem() {
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public void rename(Path src, Path dst, final Options.Rename... options) throws IOException {
+
+ }
+
+ };
+
+ alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
+ assertTrue(alluxioCarbonFile.renameForce(fileName));
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/HDFSCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/HDFSCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/HDFSCarbonFileTest.java
new file mode 100644
index 0000000..b09e14e
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/HDFSCarbonFileTest.java
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.filesystem;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Options;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
+import org.apache.hadoop.util.Progressable;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+
+public class HDFSCarbonFileTest {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(HDFSCarbonFile.class.getName());
+ private static HDFSCarbonFile hdfsCarbonFile;
+ private static FileStatus fileStatus = null;
+ private static FileStatus fileStatusWithOutDirectoryPermission;
+ private static String fileName = null;
+ private static FileSystem fs = null;
+ private static Path pt;
+
+ @BeforeClass
+ static public void setUp() throws IOException {
+ Configuration config = new Configuration();
+//adding local hadoop configuration
+ config.addResource(new Path("core-site.xml"));
+ config.addResource(new Path("hdfs-site.xml"));
+ fileName = "Test.carbondata"; //this path is HDFS path
+ pt = new Path(fileName);
+ fs = FileSystem.get(new Configuration(config));
+ fs.create(pt);
+ if (fs.exists(pt)) {
+ OutputStream os = fs.create(pt,
+ new Progressable() {
+ public void progress() {
+ LOGGER.info("Started Writing to File===");
+ }
+ });
+ BufferedWriter br = new BufferedWriter(new OutputStreamWriter(os, "UTF-8"));
+ br.write("Hello World");
+ br.close();
+ fs.close();
+
+ fileStatus = new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName));
+ fileStatusWithOutDirectoryPermission = new FileStatus(12L, false, 60, 120l, 180L, new Path(fileName));
+ hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
+
+ }
+ }
+
+ @AfterClass
+ static public void cleanUp() {
+ try {
+ fs.delete(pt, true);
+ } catch (IOException e) {
+ LOGGER.error("Exception Occured" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void testRenameForceForException() throws IOException {
+
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ throw new IOException();
+ }
+
+ };
+ hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
+ hdfsCarbonFile.renameForce(fileName);
+ }
+
+ @Test
+ public void testListFilesWithOutDirectoryPermission() {
+ hdfsCarbonFile = new HDFSCarbonFile(fileStatusWithOutDirectoryPermission);
+ new MockUp<FileStatus>() {
+ @Mock
+ public boolean isDirectory() {
+ return false;
+ }
+
+ };
+
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+
+ new MockUp<FileStatus>() {
+ @Mock
+ public Path getPath() {
+ return new Path(fileName);
+ }
+
+ };
+ assertEquals(hdfsCarbonFile.listFiles(), null);
+ }
+
+ @Test
+ public void testConstructorWithFilePath() {
+ hdfsCarbonFile = new HDFSCarbonFile(fileName);
+ assertTrue(hdfsCarbonFile instanceof HDFSCarbonFile);
+ }
+
+ @Test
+ public void testListFilesForNullListStatus() {
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public FileStatus[] listStatus(Path var1) throws IOException {
+
+ return null;
+ }
+
+ };
+ hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
+ assertEquals(hdfsCarbonFile.listFiles().length, 0);
+ }
+
+ @Test
+ public void testListDirectory() {
+ hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public FileStatus[] listStatus(Path var1) throws IOException {
+
+ FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
+ return fileStatus;
+ }
+
+ };
+
+ assertEquals(hdfsCarbonFile.listFiles().length, 1);
+ }
+
+ @Test
+ public void testListFilesForException() throws IOException {
+ new HDFSCarbonFile(fileStatusWithOutDirectoryPermission);
+
+ new MockUp<FileStatus>() {
+ @Mock
+ public Path getPath() {
+ return new Path(fileName);
+ }
+
+ };
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ throw new IOException();
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public FileStatus[] listStatus(Path var1) throws IOException {
+
+ throw new IOException();
+ }
+
+ };
+ hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
+ hdfsCarbonFile.listFiles();
+ }
+
+ @Test
+ public void testListFilesWithCarbonFilter() {
+ CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
+
+ @Override
+ public boolean accept(CarbonFile file) {
+ return true;
+ }
+ };
+ new MockUp<FileStatus>() {
+ @Mock
+ public boolean isDirectory() {
+ return true;
+ }
+
+ };
+
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+
+ new MockUp<FileStatus>() {
+ @Mock
+ public Path getPath() {
+ return new Path(fileName);
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public FileStatus[] listStatus(Path var1) throws IOException {
+
+ FileStatus fileStatus[] = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
+ return fileStatus;
+ }
+
+ };
+ hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
+ assertEquals(hdfsCarbonFile.listFiles(carbonFileFilter).length, 1);
+ }
+
+ @Test
+ public void testlistFilesWithoutFilter() {
+ CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
+
+ @Override
+ public boolean accept(CarbonFile file) {
+ return false;
+ }
+ };
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public FileStatus[] listStatus(Path var1) throws IOException {
+
+ FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
+ return fileStatus;
+ }
+
+ };
+ hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
+ assertEquals(hdfsCarbonFile.listFiles(carbonFileFilter).length, 0);
+ }
+
+ @Test
+ public void testGetParentFileForNull() {
+
+ new MockUp<Path>() {
+ @Mock
+ public Path getParent() {
+ return null;
+ }
+
+ };
+ new MockUp<FileStatus>() {
+ @Mock
+ public Path getPath() {
+ return new Path(fileName);
+ }
+
+ };
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+
+ new MockUp<FileStatus>() {
+ @Mock
+ public Path getPath() {
+ return new Path(fileName);
+ }
+
+ };
+ hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
+ assertEquals(hdfsCarbonFile.getParentFile(), null);
+ }
+
+ @Test
+ public void testGetParentFile() {
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+ new MockUp<Path>() {
+ @Mock
+ public Path getParent() {
+ return new Path(fileName);
+ }
+
+ };
+ new MockUp<FileStatus>() {
+ @Mock
+ public Path getPath() {
+ return new Path(fileName);
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public FileStatus getFileStatus(Path file) throws IOException {
+
+ return new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName));
+ }
+
+ };
+
+ hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
+ assertTrue(hdfsCarbonFile.getParentFile() instanceof CarbonFile);
+ }
+
+ @Test
+ public void testForNonDisributedSystem() {
+ new HDFSCarbonFile(fileStatus);
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new WebHdfsFileSystem();
+ }
+
+ };
+ assertEquals(hdfsCarbonFile.renameForce(fileName), false);
+ }
+
+ @Test
+ public void testrenameForceForDisributedSystem() {
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public void rename(Path src, Path dst, final Options.Rename... options) throws IOException {
+
+ }
+
+ };
+ hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
+ assertEquals(hdfsCarbonFile.renameForce(fileName), true);
+
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/LocalCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/LocalCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/LocalCarbonFileTest.java
new file mode 100644
index 0000000..9d1ab42
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/LocalCarbonFileTest.java
@@ -0,0 +1,471 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.filesystem;
+
+import mockit.Mock;
+import mockit.MockUp;
+
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import sun.nio.ch.FileChannelImpl;
+
+import java.io.*;
+import java.nio.channels.ReadableByteChannel;
+import java.util.Objects;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+
+public class LocalCarbonFileTest {
+
+ private static LocalCarbonFile localCarbonFile;
+ private static File file;
+ private static File dir;
+ private static FileOutputStream oFile;
+
+ @BeforeClass
+ static public void setUp() {
+ file = new File("Test.carbondata");
+ dir = new File("Testdir.carbondata");
+ if (!file.exists())
+ try {
+ file.createNewFile();
+ dir.mkdir();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
+ oFile = new FileOutputStream(file, true);
+
+
+ byte[] bytes = "core java api".getBytes();
+
+ oFile.write(bytes);
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ localCarbonFile = new LocalCarbonFile(file);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ @AfterClass
+ static public void cleanUp() {
+ file.delete();
+ dir.delete();
+
+ }
+
+ @Test
+ public void testListFilesWithCarbonFileFilterAndWithOutOutDirectoryPermission() {
+ CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
+ @Override
+ public boolean accept(CarbonFile file) {
+ return false;
+ }
+ };
+ new MockUp<File>() {
+ @Mock
+ public boolean isDirectory() {
+ return false;
+ }
+
+
+ };
+ assertTrue(localCarbonFile.listFiles(carbonFileFilter) == null);
+ }
+
+ @Test
+ public void testListFilesWithOutDirPermission() {
+ localCarbonFile = new LocalCarbonFile(file);
+ new MockUp<File>() {
+ @Mock
+ public boolean isDirectory() {
+ return false;
+ }
+ };
+ assertTrue(localCarbonFile.listFiles() == null);
+ }
+
+ @Test
+ public void testCreateNewFileForException() throws IOException {
+ localCarbonFile = new LocalCarbonFile(new File(""));
+ assertTrue(!localCarbonFile.createNewFile());
+ }
+
+ @Test
+ public void testCheckIfFileExists() throws IOException {
+ localCarbonFile = new LocalCarbonFile(new File(""));
+ assertTrue(!localCarbonFile.exists());
+ }
+
+ @Test
+ public void testRenameForce() {
+ localCarbonFile = new LocalCarbonFile(file);
+ assertTrue(localCarbonFile.renameForce("Testdb.carbon"));
+ File file1 = new File("Testdb.carbon");
+ if (file1.exists()) {
+ file1.delete();
+ }
+ }
+
+ @Test
+ public void testRenameTo() {
+ localCarbonFile = new LocalCarbonFile(file);
+ assertTrue(!localCarbonFile.renameTo("Testdb.carbon"));
+ }
+
+ @Test
+ public void testsetLastModifiedTime() {
+ localCarbonFile = new LocalCarbonFile(file);
+ assertTrue(!localCarbonFile.setLastModifiedTime(50L));
+ }
+
+ @Test
+ public void testtruncate() {
+ localCarbonFile = new LocalCarbonFile(file);
+ final int[] counter = {0};
+ new MockUp<FileFactory>() {
+ @Mock
+ public FileFactory.FileType getFileType(String path) {
+ {
+ return FileFactory.FileType.LOCAL;
+ }
+ }
+ };
+ new MockUp<FileFactory>() {
+ @Mock
+ public boolean isFileExist(String filePath, FileFactory.FileType fileType) throws IOException {
+ {
+ return true;
+ }
+ }
+ };
+ new MockUp<CarbonFile>() {
+ @Mock
+ boolean delete() {
+ return true;
+ }
+ };
+ new MockUp<FileFactory>() {
+ @Mock
+ public boolean createNewFile(String filePath, FileFactory.FileType fileType) throws IOException {
+ {
+ return true;
+ }
+ }
+ };
+ new MockUp<FileFactory>() {
+ @Mock
+ public CarbonFile getCarbonFile(String path, FileFactory.FileType fileType) {
+ {
+ return new LocalCarbonFile(path);
+ }
+ }
+ };
+ new MockUp<CarbonFile>() {
+ @Mock
+ boolean delete() {
+ return true;
+ }
+ };
+
+ new MockUp<FileChannelImpl>() {
+ @Mock
+ public long transferFrom(ReadableByteChannel var1, long var2, long var4) throws IOException {
+ if (counter[0] == 0) {
+ counter[0] = counter[0] + 1;
+ return 0L;
+ } else {
+ return 1L;
+ }
+ }
+ };
+ new MockUp<CarbonFile>() {
+ @Mock
+ boolean renameForce(String changetoName) {
+ return true;
+ }
+ };
+ localCarbonFile = new LocalCarbonFile(file);
+ assertTrue(localCarbonFile.truncate(file.getName(), 1L));
+ }
+
+ @Test
+ public void testtruncateForException() throws IOException {
+ localCarbonFile = new LocalCarbonFile(file);
+ new MockUp<FileFactory>() {
+ @Mock
+ public FileFactory.FileType getFileType(String path) {
+ {
+ return FileFactory.FileType.LOCAL;
+ }
+ }
+ };
+ new MockUp<FileFactory>() {
+ @Mock
+ public boolean isFileExist(String filePath, FileFactory.FileType fileType) throws IOException {
+ {
+ return true;
+ }
+ }
+ };
+ new MockUp<FileFactory>() {
+ @Mock
+ public CarbonFile getCarbonFile(String path, FileFactory.FileType fileType) {
+ {
+ return new LocalCarbonFile(path);
+ }
+ }
+ };
+ new MockUp<CarbonFile>() {
+ @Mock
+ boolean delete() {
+ return true;
+ }
+ };
+ new MockUp<FileFactory>() {
+ @Mock
+ public boolean createNewFile(String filePath, FileFactory.FileType fileType) throws IOException {
+ {
+ throw new IOException();
+ }
+ }
+ };
+
+
+ localCarbonFile.truncate(file.getName(), 2L);
+ }
+
+ @Test
+ public void testListFilesWithDirPermission() {
+ localCarbonFile = new LocalCarbonFile(file);
+ new MockUp<File>() {
+ @Mock
+ public boolean isDirectory() {
+ return true;
+ }
+ };
+ new MockUp<File>() {
+ @Mock
+ public File[] listFiles() {
+ return null;
+ }
+
+
+ };
+ localCarbonFile = new LocalCarbonFile(dir);
+ assertTrue(localCarbonFile.listFiles().length == 0);
+ }
+
+ @Test
+ public void testListFilesWithCarbonFileFilterAndDirectoryPermission() {
+ CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
+ @Override
+ public boolean accept(CarbonFile file) {
+ return true;
+ }
+ };
+ new MockUp<File>() {
+ @Mock
+ public boolean isDirectory() {
+ return true;
+ }
+ };
+ new MockUp<File>() {
+ @Mock
+ public File[] listFiles(FileFilter filter) {
+
+ File[] file = new File[]{dir};
+ return file;
+ }
+
+
+ };
+
+ localCarbonFile = new LocalCarbonFile(dir);
+
+ assertTrue(localCarbonFile.listFiles(carbonFileFilter).length == 1);
+ }
+
+ @Test
+ public void testListFilesForNullWithCarbonFileFilterAndDirectoryPermission() {
+ CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
+ @Override
+ public boolean accept(CarbonFile file) {
+ return true;
+ }
+ };
+ new MockUp<File>() {
+ @Mock
+ public File[] listFiles(FileFilter filter) {
+ return null;
+ }
+
+
+ };
+ new MockUp<File>() {
+ @Mock
+ public boolean isDirectory() {
+ return false;
+ }
+
+ @Mock
+ public File[] listFiles(FileFilter filter) {
+ return null;
+ }
+
+
+ };
+ localCarbonFile = new LocalCarbonFile(dir);
+
+ assertTrue(localCarbonFile.listFiles(carbonFileFilter) == null);
+ }
+
+ @Test
+ public void testListFilesForEmptyFileArrayWithCarbonFileFilter() {
+ CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
+ @Override
+ public boolean accept(CarbonFile file) {
+ return true;
+ }
+ };
+ new MockUp<CarbonFileFilter>() {
+ @Mock
+ boolean accept(CarbonFile file) {
+ return true;
+ }
+ };
+ new MockUp<File>() {
+ @Mock
+ public File[] listFiles(FileFilter filter) {
+ return null;
+ }
+ };
+ new MockUp<File>() {
+ @Mock
+ public boolean isDirectory() {
+ return true;
+ }
+
+ @Mock
+ public File[] listFiles(FileFilter filter) {
+ return null;
+ }
+ };
+ localCarbonFile = new LocalCarbonFile(dir);
+
+ assertTrue(localCarbonFile.listFiles(carbonFileFilter).length == 0);
+ }
+
+ @Test
+ public void testFilesForConnicalPath() {
+
+ new MockUp<File>() {
+ @Mock
+ public String getCanonicalPath() throws IOException {
+ return "testFile";
+ }
+
+
+ };
+ localCarbonFile = new LocalCarbonFile(dir);
+
+ assertTrue(Objects.equals(localCarbonFile.getCanonicalPath(), "testFile"));
+ }
+
+ @Test
+ public void testFilesForConnicalPathException() throws IOException {
+
+ new MockUp<File>() {
+ @Mock
+ public String getCanonicalPath() throws IOException {
+ throw new IOException();
+ }
+
+
+ };
+ localCarbonFile = new LocalCarbonFile(dir);
+
+ localCarbonFile.getCanonicalPath();
+ }
+
+ @Test
+ public void testFilesForAbsolutePath() {
+
+ new MockUp<File>() {
+ @Mock
+ public String getAbsolutePath() {
+ return "testFile";
+ }
+
+
+ };
+ localCarbonFile = new LocalCarbonFile(dir);
+
+ assertEquals(localCarbonFile.getAbsolutePath(), "testFile");
+ }
+
+ @Test
+ public void testFilesForGetPath() {
+
+ new MockUp<File>() {
+ @Mock
+ public String getPath() {
+ return "testFile";
+ }
+
+
+ };
+ localCarbonFile = new LocalCarbonFile(dir);
+
+ assertEquals(localCarbonFile.getPath(), "testFile");
+ }
+
+ @Test
+ public void testFilesForFileExists() {
+
+ localCarbonFile = new LocalCarbonFile(new File(""));
+ assertEquals(localCarbonFile.exists(), false);
+ }
+
+ @Test
+ public void testRenameForceForFileNotExists() {
+ new MockUp<File>() {
+ @Mock
+ public boolean exists() {
+ return false;
+ }
+
+ @Mock
+ public boolean renameTo(File dest) {
+ return true;
+ }
+ };
+
+ localCarbonFile = new LocalCarbonFile("demo.txt");
+
+ assertEquals(localCarbonFile.renameForce("Test.carbondata"), true);
+ }
+}
[13/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/resolver/ConditionalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/ConditionalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/ConditionalFilterResolverImpl.java
deleted file mode 100644
index 82c6b84..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/ConditionalFilterResolverImpl.java
+++ /dev/null
@@ -1,229 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.resolver;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.SortedMap;
-
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.core.metadata.Encoding;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.conditional.BinaryConditionalExpression;
-import org.apache.carbondata.scan.expression.conditional.ConditionalExpression;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.FilterUtil;
-import org.apache.carbondata.scan.filter.intf.FilterExecuterType;
-import org.apache.carbondata.scan.filter.resolver.metadata.FilterResolverMetadata;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.visitor.FilterInfoTypeVisitorFactory;
-
-public class ConditionalFilterResolverImpl implements FilterResolverIntf {
-
- private static final long serialVersionUID = 1838955268462201691L;
- protected Expression exp;
- protected boolean isExpressionResolve;
- protected boolean isIncludeFilter;
- private DimColumnResolvedFilterInfo dimColResolvedFilterInfo;
-
- public ConditionalFilterResolverImpl(Expression exp, boolean isExpressionResolve,
- boolean isIncludeFilter) {
- this.exp = exp;
- this.isExpressionResolve = isExpressionResolve;
- this.isIncludeFilter = isIncludeFilter;
- this.dimColResolvedFilterInfo = new DimColumnResolvedFilterInfo();
- }
-
- /**
- * This API will resolve the filter expression and generates the
- * dictionaries for executing/evaluating the filter expressions in the
- * executer layer.
- *
- * @throws FilterUnsupportedException
- */
- @Override public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier)
- throws FilterUnsupportedException, IOException {
- FilterResolverMetadata metadata = new FilterResolverMetadata();
- metadata.setTableIdentifier(absoluteTableIdentifier);
- if ((!isExpressionResolve) && exp instanceof BinaryConditionalExpression) {
- BinaryConditionalExpression binaryConditionalExpression = (BinaryConditionalExpression) exp;
- Expression leftExp = binaryConditionalExpression.getLeft();
- Expression rightExp = binaryConditionalExpression.getRight();
- if (leftExp instanceof ColumnExpression) {
- ColumnExpression columnExpression = (ColumnExpression) leftExp;
- metadata.setColumnExpression(columnExpression);
- metadata.setExpression(rightExp);
- metadata.setIncludeFilter(isIncludeFilter);
- // If imei=imei comes in filter condition then we need to
- // skip processing of right expression.
- // This flow has reached here assuming that this is a single
- // column expression.
- // we need to check if the other expression contains column
- // expression or not in depth.
- CarbonDimension dimension = columnExpression.getDimension();
- if (FilterUtil.checkIfExpressionContainsColumn(rightExp)
- || FilterUtil.isExpressionNeedsToResolved(rightExp, isIncludeFilter) &&
- dimension.hasEncoding(Encoding.DICTIONARY) && !dimension
- .hasEncoding(Encoding.DIRECT_DICTIONARY)) {
- isExpressionResolve = true;
- } else {
- //Visitor pattern is been used in this scenario inorder to populate the
- // dimColResolvedFilterInfo
- //visitable object with filter member values based on the visitor type, currently there
- //3 types of visitors custom,direct and no dictionary, all types of visitor populate
- //the visitable instance as per its buisness logic which is different for all the
- // visitors.
- dimColResolvedFilterInfo.populateFilterInfoBasedOnColumnType(
- FilterInfoTypeVisitorFactory.getResolvedFilterInfoVisitor(columnExpression),
- metadata);
- }
- } else if (rightExp instanceof ColumnExpression) {
- ColumnExpression columnExpression = (ColumnExpression) rightExp;
- metadata.setColumnExpression(columnExpression);
- metadata.setExpression(leftExp);
- metadata.setIncludeFilter(isIncludeFilter);
- if (columnExpression.getDataType().equals(DataType.TIMESTAMP) ||
- columnExpression.getDataType().equals(DataType.DATE)) {
- isExpressionResolve = true;
- } else {
- // if imei=imei comes in filter condition then we need to
- // skip processing of right expression.
- // This flow has reached here assuming that this is a single
- // column expression.
- // we need to check if the other expression contains column
- // expression or not in depth.
- if (FilterUtil.checkIfExpressionContainsColumn(leftExp)) {
- isExpressionResolve = true;
- } else {
-
- dimColResolvedFilterInfo.populateFilterInfoBasedOnColumnType(
- FilterInfoTypeVisitorFactory.getResolvedFilterInfoVisitor(columnExpression),
- metadata);
-
- }
- }
- } else {
- isExpressionResolve = true;
- }
- }
- if (isExpressionResolve && exp instanceof ConditionalExpression) {
- ConditionalExpression conditionalExpression = (ConditionalExpression) exp;
- List<ColumnExpression> columnList = conditionalExpression.getColumnList();
- metadata.setColumnExpression(columnList.get(0));
- metadata.setExpression(exp);
- metadata.setIncludeFilter(isIncludeFilter);
- if (!columnList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY) ||
- columnList.get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
- dimColResolvedFilterInfo.populateFilterInfoBasedOnColumnType(
- FilterInfoTypeVisitorFactory.getResolvedFilterInfoVisitor(columnList.get(0)), metadata);
- } else if (columnList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY) &&
- !(columnList.get(0).getDimension().getDataType() == DataType.STRUCT ||
- columnList.get(0).getDimension().getDataType() == DataType.ARRAY)) {
- dimColResolvedFilterInfo.setFilterValues(
- FilterUtil.getFilterListForAllValues(absoluteTableIdentifier, exp, columnList.get(0),
- isIncludeFilter));
-
- dimColResolvedFilterInfo.setColumnIndex(columnList.get(0).getDimension().getOrdinal());
- dimColResolvedFilterInfo.setDimension(columnList.get(0).getDimension());
- }
- }
-
- }
-
- /**
- * Left node will not be presentin this scenario
- *
- * @return left node of type FilterResolverIntf instance
- */
- public FilterResolverIntf getLeft() {
- // TODO Auto-generated method stub
- return null;
- }
-
- /**
- * Right node will not be presentin this scenario
- *
- * @return left node of type FilterResolverIntf instance
- */
- @Override public FilterResolverIntf getRight() {
- // TODO Auto-generated method stub
- return null;
- }
-
- /**
- * Method will return the DimColumnResolvedFilterInfo instance which consists
- * the mapping of the respective dimension and its surrogates involved in
- * filter expression.
- *
- * @return DimColumnResolvedFilterInfo
- */
- public DimColumnResolvedFilterInfo getDimColResolvedFilterInfo() {
- return dimColResolvedFilterInfo;
- }
-
- /**
- * method will calculates the start key based on the filter surrogates
- */
- public void getStartKey(long[] startKey,
- SortedMap<Integer, byte[]> setOfStartKeyByteArray, List<long[]> startKeyList) {
- FilterUtil.getStartKey(dimColResolvedFilterInfo.getDimensionResolvedFilterInstance(),
- startKey, startKeyList);
- FilterUtil.getStartKeyForNoDictionaryDimension(dimColResolvedFilterInfo,
- setOfStartKeyByteArray);
- }
-
- /**
- * get the start key based on the filter surrogates
- */
- @Override public void getEndKey(SegmentProperties segmentProperties, long[] endKeys,
- SortedMap<Integer, byte[]> setOfEndKeyByteArray, List<long[]> endKeyList) {
- FilterUtil.getEndKey(dimColResolvedFilterInfo.getDimensionResolvedFilterInstance(), endKeys,
- segmentProperties, endKeyList);
- FilterUtil.getEndKeyForNoDictionaryDimension(dimColResolvedFilterInfo,
- setOfEndKeyByteArray);
- }
-
- /**
- * Method will return the executer type for particular conditional resolver
- * basically two types of executers will be formed for the conditional query.
- *
- * @return the filter executer type
- */
- @Override public FilterExecuterType getFilterExecuterType() {
- switch (exp.getFilterExpressionType()) {
- case NOT_EQUALS:
- case NOT_IN:
- return FilterExecuterType.EXCLUDE;
-
- default:
- return FilterExecuterType.INCLUDE;
- }
-
- }
-
- @Override public Expression getFilterExpression() {
- // TODO Auto-generated method stub
- return exp;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/resolver/FilterResolverIntf.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/FilterResolverIntf.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/FilterResolverIntf.java
deleted file mode 100644
index a04984b..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/FilterResolverIntf.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.resolver;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.List;
-import java.util.SortedMap;
-
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.intf.FilterExecuterType;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-
-public interface FilterResolverIntf extends Serializable {
-
- /**
- * This API will resolve the filter expression and generates the
- * dictionaries for executing/evaluating the filter expressions in the
- * executer layer.
- *
- * @throws IOException
- * @throws FilterUnsupportedException
- */
- void resolve(AbsoluteTableIdentifier absoluteTableIdentifier)
- throws IOException, FilterUnsupportedException;
-
- /**
- * This API will provide the left column filter expression
- * inorder to resolve the left expression filter.
- *
- * @return FilterResolverIntf
- */
- FilterResolverIntf getLeft();
-
- /**
- * API will provide the right column filter expression inorder to resolve
- * the right expression filter.
- *
- * @return FilterResolverIntf
- */
- FilterResolverIntf getRight();
-
- /**
- * API will return the resolved filter instance, this instance will provide
- * the resolved surrogates based on the applied filter
- *
- * @return DimColumnResolvedFilterInfo object
- */
- DimColumnResolvedFilterInfo getDimColResolvedFilterInfo();
-
- /**
- * API will get the start key based on the filter applied based on the key generator
- *
- * @param startKey
- * @param setOfStartKeyByteArray
- */
- void getStartKey(long[] startKey, SortedMap<Integer, byte[]> setOfStartKeyByteArray,
- List<long[]> startKeyList);
-
- /**
- * API will read the end key based on the max surrogate of
- * particular dimension column
- *
- * @param endKeys
- * @param setOfEndKeyByteArray
- * @return
- */
- void getEndKey(SegmentProperties segmentProperties, long[] endKeys,
- SortedMap<Integer, byte[]> setOfEndKeyByteArray, List<long[]> endKeyList);
-
- /**
- * API will return the filter executer type which will be used to evaluate
- * the resolved filter while query execution
- *
- * @return FilterExecuterType.
- */
- FilterExecuterType getFilterExecuterType();
-
- Expression getFilterExpression();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/resolver/LogicalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/LogicalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/LogicalFilterResolverImpl.java
deleted file mode 100644
index 786c2d4..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/LogicalFilterResolverImpl.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.filter.resolver;
-
-import java.util.List;
-import java.util.SortedMap;
-
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.scan.expression.BinaryExpression;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.filter.intf.ExpressionType;
-import org.apache.carbondata.scan.filter.intf.FilterExecuterType;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-
-public class LogicalFilterResolverImpl implements FilterResolverIntf {
- /**
- *
- */
- private static final long serialVersionUID = 5734382980564402914L;
-
- protected FilterResolverIntf leftEvalutor;
-
- protected FilterResolverIntf rightEvalutor;
-
- protected ExpressionType filterExpressionType;
-
- private BinaryExpression filterExpression;
-
- public LogicalFilterResolverImpl(FilterResolverIntf leftEvalutor,
- FilterResolverIntf rightEvalutor, BinaryExpression currentExpression) {
- this.leftEvalutor = leftEvalutor;
- this.rightEvalutor = rightEvalutor;
- this.filterExpressionType = currentExpression.getFilterExpressionType();
- this.filterExpression = currentExpression;
- }
-
- /**
- * Logical filter resolver will return the left and right filter expresison
- * node for filter evaluation, so in this instance no implementation is required.
- *
- * @param absoluteTableIdentifier
- */
- @Override public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier) {
-
- }
-
- /**
- * Since its a binary condition expresion the getLeft method will get the left
- * node of filter expression
- *
- * @return FilterResolverIntf.
- */
- public FilterResolverIntf getLeft() {
- return leftEvalutor;
- }
-
- /**
- * Since its a binary condition expresion the getRight method will get the left
- * node of filter expression
- *
- * @return FilterResolverIntf.
- */
- public FilterResolverIntf getRight() {
- return rightEvalutor;
- }
-
- @Override public DimColumnResolvedFilterInfo getDimColResolvedFilterInfo() {
- return null;
- }
-
- @Override
- public void getStartKey(long[] startKey, SortedMap<Integer, byte[]> setOfStartKeyByteArray,
- List<long[]> startKeyList) {
-
- }
-
- @Override public void getEndKey(SegmentProperties segmentProperties, long[] endKeys,
- SortedMap<Integer, byte[]> setOfEndKeyByteArray, List<long[]> endKeyList) {
-
- }
-
- @Override public FilterExecuterType getFilterExecuterType() {
- switch (filterExpressionType) {
- case OR:
- return FilterExecuterType.OR;
- case AND:
- return FilterExecuterType.AND;
-
- default:
- return null;
- }
- }
-
- @Override public Expression getFilterExpression() {
- return filterExpression;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelFilterResolverImpl.java
deleted file mode 100644
index bad43ba..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelFilterResolverImpl.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.filter.resolver;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.conditional.ConditionalExpression;
-import org.apache.carbondata.scan.filter.intf.FilterExecuterType;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-
-public class RowLevelFilterResolverImpl extends ConditionalFilterResolverImpl {
-
- private static final long serialVersionUID = 176122729713729929L;
-
- private List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList;
- private List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList;
- private AbsoluteTableIdentifier tableIdentifier;
-
- public RowLevelFilterResolverImpl(Expression exp, boolean isExpressionResolve,
- boolean isIncludeFilter, AbsoluteTableIdentifier tableIdentifier) {
- super(exp, isExpressionResolve, isIncludeFilter);
- dimColEvaluatorInfoList =
- new ArrayList<DimColumnResolvedFilterInfo>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- msrColEvalutorInfoList = new ArrayList<MeasureColumnResolvedFilterInfo>(
- CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- this.tableIdentifier = tableIdentifier;
- }
-
- /**
- * Method which will resolve the filter expression by converting the filter member
- * to its assigned dictionary values.
- */
- public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier) {
- DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = null;
- MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo = null;
- int index = 0;
- if (exp instanceof ConditionalExpression) {
- ConditionalExpression conditionalExpression = (ConditionalExpression) exp;
- List<ColumnExpression> columnList = conditionalExpression.getColumnList();
- for (ColumnExpression columnExpression : columnList) {
- if (columnExpression.isDimension()) {
- dimColumnEvaluatorInfo = new DimColumnResolvedFilterInfo();
- dimColumnEvaluatorInfo.setColumnIndex(columnExpression.getCarbonColumn().getOrdinal());
- dimColumnEvaluatorInfo.setRowIndex(index++);
- dimColumnEvaluatorInfo.setDimension(columnExpression.getDimension());
- dimColumnEvaluatorInfo.setDimensionExistsInCurrentSilce(false);
- dimColEvaluatorInfoList.add(dimColumnEvaluatorInfo);
- } else {
- msrColumnEvalutorInfo = new MeasureColumnResolvedFilterInfo();
- msrColumnEvalutorInfo.setRowIndex(index++);
- msrColumnEvalutorInfo
- .setColumnIndex(columnExpression.getCarbonColumn().getOrdinal());
- msrColumnEvalutorInfo.setType(columnExpression.getCarbonColumn().getDataType());
- msrColEvalutorInfoList.add(msrColumnEvalutorInfo);
- }
- }
- }
- }
-
- /**
- * This method will provide the executer type to the callee inorder to identify
- * the executer type for the filter resolution, Row level filter executer is a
- * special executer since it get all the rows of the specified filter dimension
- * and will be send to the spark for processing
- */
- @Override public FilterExecuterType getFilterExecuterType() {
- return FilterExecuterType.ROWLEVEL;
- }
-
- /**
- * Method will the read filter expression corresponding to the resolver.
- * This method is required in row level executer inorder to evaluate the filter
- * expression against spark, as mentioned above row level is a special type
- * filter resolver.
- *
- * @return Expression
- */
- public Expression getFilterExpresion() {
- return exp;
- }
-
- /**
- * Method will return the DimColumnResolvedFilterInfo instance which consists
- * the mapping of the respective dimension and its surrogates involved in
- * filter expression.
- *
- * @return DimColumnResolvedFilterInfo
- */
- public List<DimColumnResolvedFilterInfo> getDimColEvaluatorInfoList() {
- return dimColEvaluatorInfoList;
- }
-
- /**
- * Method will return the DimColumnResolvedFilterInfo instance which containts
- * measure level details.
- *
- * @return MeasureColumnResolvedFilterInfo
- */
- public List<MeasureColumnResolvedFilterInfo> getMsrColEvalutorInfoList() {
- return msrColEvalutorInfoList;
- }
-
- /**
- * Method will return table information which will be required for retrieving
- * dictionary cache inorder to read all the members of respective dimension.
- *
- * @return AbsoluteTableIdentifier
- */
- public AbsoluteTableIdentifier getTableIdentifier() {
- return tableIdentifier;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
deleted file mode 100644
index 54b4e96..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
+++ /dev/null
@@ -1,266 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.filter.resolver;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.SortedMap;
-
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.metadata.Encoding;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
-import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
-import org.apache.carbondata.core.util.ByteUtil;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.conditional.BinaryConditionalExpression;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.expression.logical.BinaryLogicalExpression;
-import org.apache.carbondata.scan.filter.DimColumnFilterInfo;
-import org.apache.carbondata.scan.filter.FilterUtil;
-import org.apache.carbondata.scan.filter.intf.FilterExecuterType;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
-
-public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverImpl {
-
- /**
- *
- */
- private static final long serialVersionUID = 6629319265336666789L;
- private List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList;
- private List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList;
- private AbsoluteTableIdentifier tableIdentifier;
-
- public RowLevelRangeFilterResolverImpl(Expression exp, boolean isExpressionResolve,
- boolean isIncludeFilter, AbsoluteTableIdentifier tableIdentifier) {
- super(exp, isExpressionResolve, isIncludeFilter);
- dimColEvaluatorInfoList =
- new ArrayList<DimColumnResolvedFilterInfo>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- msrColEvalutorInfoList = new ArrayList<MeasureColumnResolvedFilterInfo>(
- CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- this.tableIdentifier = tableIdentifier;
- }
-
- /**
- * This method will return the filter values which is present in the range level
- * conditional expressions.
- *
- * @return
- */
- public byte[][] getFilterRangeValues(SegmentProperties segmentProperties) {
-
- if (null != dimColEvaluatorInfoList.get(0).getFilterValues() && !dimColEvaluatorInfoList.get(0)
- .getDimension().hasEncoding(Encoding.DICTIONARY)) {
- List<byte[]> noDictFilterValuesList =
- dimColEvaluatorInfoList.get(0).getFilterValues().getNoDictionaryFilterValuesList();
- return noDictFilterValuesList.toArray((new byte[noDictFilterValuesList.size()][]));
- } else if (null != dimColEvaluatorInfoList.get(0).getFilterValues() && dimColEvaluatorInfoList
- .get(0).getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
- return FilterUtil.getKeyArray(this.dimColEvaluatorInfoList.get(0).getFilterValues(),
- this.dimColEvaluatorInfoList.get(0).getDimension(),
- segmentProperties);
- }
- return null;
-
- }
-
- /**
- * method will get the start key based on the filter surrogates
- *
- * @return start IndexKey
- */
- public void getStartKey(long[] startKey,
- SortedMap<Integer, byte[]> noDictStartKeys, List<long[]> startKeyList) {
- FilterUtil.getStartKey(dimColEvaluatorInfoList.get(0).getDimensionResolvedFilterInstance(),
- startKey, startKeyList);
- FilterUtil
- .getStartKeyForNoDictionaryDimension(dimColEvaluatorInfoList.get(0), noDictStartKeys);
- }
-
- /**
- * method will get the start key based on the filter surrogates
- *
- * @return end IndexKey
- */
- @Override public void getEndKey(SegmentProperties segmentProperties, long[] endKeys,
- SortedMap<Integer, byte[]> noDicEndKeys, List<long[]> endKeyList) {
- FilterUtil.getEndKey(dimColEvaluatorInfoList.get(0).getDimensionResolvedFilterInstance(),
- endKeys, segmentProperties, endKeyList);
- FilterUtil
- .getEndKeyForNoDictionaryDimension(dimColEvaluatorInfoList.get(0), noDicEndKeys);
- }
-
- private List<byte[]> getNoDictionaryRangeValues() {
- List<ExpressionResult> listOfExpressionResults = new ArrayList<ExpressionResult>(20);
- if (this.getFilterExpression() instanceof BinaryConditionalExpression) {
- listOfExpressionResults =
- ((BinaryConditionalExpression) this.getFilterExpression()).getLiterals();
- }
- List<byte[]> filterValuesList = new ArrayList<byte[]>(20);
- boolean invalidRowsPresent = false;
- for (ExpressionResult result : listOfExpressionResults) {
- try {
- if (result.getString() == null) {
- filterValuesList.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL.getBytes());
- continue;
- }
- filterValuesList.add(result.getString().getBytes());
- } catch (FilterIllegalMemberException e) {
- // Any invalid member while evaluation shall be ignored, system will log the
- // error only once since all rows the evaluation happens so inorder to avoid
- // too much log inforation only once the log will be printed.
- FilterUtil.logError(e, invalidRowsPresent);
- }
- }
- Comparator<byte[]> filterNoDictValueComaparator = new Comparator<byte[]>() {
- @Override public int compare(byte[] filterMember1, byte[] filterMember2) {
- return ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterMember1, filterMember2);
- }
-
- };
- Collections.sort(filterValuesList, filterNoDictValueComaparator);
- return filterValuesList;
- }
-
- /**
- * Method which will resolve the filter expression by converting the filter
- * member to its assigned dictionary values.
- */
- public void resolve(AbsoluteTableIdentifier absoluteTableIdentifier)
- throws FilterUnsupportedException {
- DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = null;
- MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo = null;
- int index = 0;
- if (exp instanceof BinaryLogicalExpression) {
- BinaryLogicalExpression conditionalExpression = (BinaryLogicalExpression) exp;
- List<ColumnExpression> columnList = conditionalExpression.getColumnList();
- for (ColumnExpression columnExpression : columnList) {
- if (columnExpression.isDimension()) {
- dimColumnEvaluatorInfo = new DimColumnResolvedFilterInfo();
- DimColumnFilterInfo filterInfo = new DimColumnFilterInfo();
- dimColumnEvaluatorInfo.setColumnIndex(columnExpression.getCarbonColumn().getOrdinal());
- dimColumnEvaluatorInfo.setRowIndex(index++);
- dimColumnEvaluatorInfo.setDimension(columnExpression.getDimension());
- dimColumnEvaluatorInfo.setDimensionExistsInCurrentSilce(false);
- if (columnExpression.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
- filterInfo.setFilterList(getDirectSurrogateValues(columnExpression));
- } else {
- filterInfo.setFilterListForNoDictionaryCols(getNoDictionaryRangeValues());
- }
- filterInfo.setIncludeFilter(isIncludeFilter);
- dimColumnEvaluatorInfo.setFilterValues(filterInfo);
- dimColumnEvaluatorInfo
- .addDimensionResolvedFilterInstance(columnExpression.getDimension(), filterInfo);
- dimColEvaluatorInfoList.add(dimColumnEvaluatorInfo);
- } else {
- msrColumnEvalutorInfo = new MeasureColumnResolvedFilterInfo();
- msrColumnEvalutorInfo.setRowIndex(index++);
- msrColumnEvalutorInfo
- .setColumnIndex(columnExpression.getCarbonColumn().getOrdinal());
- msrColumnEvalutorInfo.setType(columnExpression.getCarbonColumn().getDataType());
- msrColEvalutorInfoList.add(msrColumnEvalutorInfo);
- }
- }
- }
- }
-
- private List<Integer> getDirectSurrogateValues(ColumnExpression columnExpression)
- throws FilterUnsupportedException {
- List<ExpressionResult> listOfExpressionResults = new ArrayList<ExpressionResult>(20);
- DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
- .getDirectDictionaryGenerator(columnExpression.getDimension().getDataType());
-
- if (this.getFilterExpression() instanceof BinaryConditionalExpression) {
- listOfExpressionResults =
- ((BinaryConditionalExpression) this.getFilterExpression()).getLiterals();
- }
- List<Integer> filterValuesList = new ArrayList<Integer>(20);
- try {
- // if any filter member provided by user is invalid throw error else
- // system can display inconsistent result.
- for (ExpressionResult result : listOfExpressionResults) {
- filterValuesList.add(directDictionaryGenerator
- .generateDirectSurrogateKey(result.getString(),
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
- }
- } catch (FilterIllegalMemberException e) {
- throw new FilterUnsupportedException(e);
- }
- return filterValuesList;
- }
-
- /**
- * Method will return the DimColumnResolvedFilterInfo instance which consists
- * the mapping of the respective dimension and its surrogates involved in
- * filter expression.
- *
- * @return DimColumnResolvedFilterInfo
- */
- public List<DimColumnResolvedFilterInfo> getDimColEvaluatorInfoList() {
- return dimColEvaluatorInfoList;
- }
-
- /**
- * Method will return the DimColumnResolvedFilterInfo instance which containts
- * measure level details.
- *
- * @return MeasureColumnResolvedFilterInfo
- */
- public List<MeasureColumnResolvedFilterInfo> getMsrColEvalutorInfoList() {
- return msrColEvalutorInfoList;
- }
-
- public AbsoluteTableIdentifier getTableIdentifier() {
- return tableIdentifier;
- }
-
- public Expression getFilterExpression() {
- return this.exp;
- }
-
- /**
- * This method will provide the executer type to the callee inorder to identify
- * the executer type for the filter resolution, Row level filter executer is a
- * special executer since it get all the rows of the specified filter dimension
- * and will be send to the spark for processing
- */
- public FilterExecuterType getFilterExecuterType() {
- switch (exp.getFilterExpressionType()) {
- case GREATERTHAN:
- return FilterExecuterType.ROWLEVEL_GREATERTHAN;
- case GREATERTHAN_EQUALTO:
- return FilterExecuterType.ROWLEVEL_GREATERTHAN_EQUALTO;
- case LESSTHAN:
- return FilterExecuterType.ROWLEVEL_LESSTHAN;
- case LESSTHAN_EQUALTO:
- return FilterExecuterType.ROWLEVEL_LESSTHAN_EQUALTO;
-
- default:
- return FilterExecuterType.ROWLEVEL;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/resolver/metadata/FilterResolverMetadata.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/metadata/FilterResolverMetadata.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/metadata/FilterResolverMetadata.java
deleted file mode 100644
index 1a1cc1d..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/metadata/FilterResolverMetadata.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.resolver.metadata;
-
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.Expression;
-
-public class FilterResolverMetadata {
- private AbsoluteTableIdentifier tableIdentifier;
- private Expression expression;
- private ColumnExpression columnExpression;
- private boolean isIncludeFilter;
-
- public AbsoluteTableIdentifier getTableIdentifier() {
- return tableIdentifier;
- }
-
- public void setTableIdentifier(AbsoluteTableIdentifier tableIdentifier) {
- this.tableIdentifier = tableIdentifier;
- }
-
- public Expression getExpression() {
- return expression;
- }
-
- public void setExpression(Expression expression) {
- this.expression = expression;
- }
-
- public ColumnExpression getColumnExpression() {
- return columnExpression;
- }
-
- public void setColumnExpression(ColumnExpression columnExpression) {
- this.columnExpression = columnExpression;
- }
-
- public boolean isIncludeFilter() {
- return isIncludeFilter;
- }
-
- public void setIncludeFilter(boolean isIncludeFilter) {
- this.isIncludeFilter = isIncludeFilter;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/DimColumnResolvedFilterInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/DimColumnResolvedFilterInfo.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/DimColumnResolvedFilterInfo.java
deleted file mode 100644
index ec59f24..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/DimColumnResolvedFilterInfo.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.filter.resolver.resolverinfo;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.DimColumnFilterInfo;
-import org.apache.carbondata.scan.filter.resolver.metadata.FilterResolverMetadata;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.visitor.ResolvedFilterInfoVisitorIntf;
-
-public class DimColumnResolvedFilterInfo implements Serializable {
- /**
- *
- */
- private static final long serialVersionUID = 3428115141211084114L;
-
- /**
- * column index in file
- */
- private int columnIndex = -1;
-
- /**
- * rowIndex
- */
- private int rowIndex = -1;
-
- private boolean isDimensionExistsInCurrentSilce = true;
-
- private String defaultValue;
-
- private CarbonDimension dimension;
-
- /**
- * reolved filter object of a particlar filter Expression.
- */
- private DimColumnFilterInfo resolvedFilterValueObj;
-
- private Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionResolvedFilter;
-
- public DimColumnResolvedFilterInfo() {
- dimensionResolvedFilter = new HashMap<CarbonDimension, List<DimColumnFilterInfo>>(20);
- }
-
- public void addDimensionResolvedFilterInstance(CarbonDimension dimension,
- DimColumnFilterInfo filterResolvedObj) {
- List<DimColumnFilterInfo> currentVals = dimensionResolvedFilter.get(dimension);
- if (null == currentVals) {
- currentVals = new ArrayList<DimColumnFilterInfo>(20);
- currentVals.add(filterResolvedObj);
- dimensionResolvedFilter.put(dimension, currentVals);
- } else {
- currentVals.add(filterResolvedObj);
- }
- }
-
- public Map<CarbonDimension, List<DimColumnFilterInfo>> getDimensionResolvedFilterInstance() {
- return dimensionResolvedFilter;
- }
-
- public CarbonDimension getDimension() {
- return dimension;
- }
-
- public void setDimension(CarbonDimension dimension) {
- this.dimension = dimension;
- }
-
- public int getColumnIndex() {
- return columnIndex;
- }
-
- public void setColumnIndex(int columnIndex) {
- this.columnIndex = columnIndex;
- }
-
- public DimColumnFilterInfo getFilterValues() {
- return resolvedFilterValueObj;
- }
-
- public void setFilterValues(final DimColumnFilterInfo resolvedFilterValueObj) {
- this.resolvedFilterValueObj = resolvedFilterValueObj;
- }
-
- public int getRowIndex() {
- return rowIndex;
- }
-
- public void setRowIndex(int rowIndex) {
- this.rowIndex = rowIndex;
- }
-
- public boolean isDimensionExistsInCurrentSilce() {
- return isDimensionExistsInCurrentSilce;
- }
-
- public void setDimensionExistsInCurrentSilce(boolean isDimensionExistsInCurrentSilce) {
- this.isDimensionExistsInCurrentSilce = isDimensionExistsInCurrentSilce;
- }
-
- public String getDefaultValue() {
- return defaultValue;
- }
-
- public void setDefaultValue(String defaultValue) {
- this.defaultValue = defaultValue;
- }
-
- public void populateFilterInfoBasedOnColumnType(ResolvedFilterInfoVisitorIntf visitor,
- FilterResolverMetadata metadata) throws FilterUnsupportedException, IOException {
- if (null != visitor) {
- visitor.populateFilterResolvedInfo(this, metadata);
- this.addDimensionResolvedFilterInstance(metadata.getColumnExpression().getDimension(),
- this.getFilterValues());
- this.setDimension(metadata.getColumnExpression().getDimension());
- this.setColumnIndex(metadata.getColumnExpression().getDimension().getOrdinal());
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java
deleted file mode 100644
index 32505fb..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/MeasureColumnResolvedFilterInfo.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.filter.resolver.resolverinfo;
-
-import java.io.Serializable;
-
-import org.apache.carbondata.core.metadata.DataType;
-
-public class MeasureColumnResolvedFilterInfo implements Serializable {
-
- private static final long serialVersionUID = 4222568289115151561L;
-
- private int columnIndex = -1;
-
- private int rowIndex = -1;
-
- private DataType type;
-
- public int getColumnIndex() {
- return columnIndex;
- }
-
- public void setColumnIndex(int columnIndex) {
- this.columnIndex = columnIndex;
- }
-
- public int getRowIndex() {
- return rowIndex;
- }
-
- public void setRowIndex(int rowIndex) {
- this.rowIndex = rowIndex;
- }
-
- public DataType getType() {
- return type;
- }
-
- public void setType(DataType dataType) {
- this.type = dataType;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
deleted file mode 100644
index 9af4b71..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.resolver.resolverinfo.visitor;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
-import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
-import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.DimColumnFilterInfo;
-import org.apache.carbondata.scan.filter.resolver.metadata.FilterResolverMetadata;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-
-public class CustomTypeDictionaryVisitor implements ResolvedFilterInfoVisitorIntf {
-
- /**
- * This Visitor method is been used to resolve or populate the filter details
- * by using custom type dictionary value, the filter membrers will be resolved using
- * custom type function which will generate dictionary for the direct column type filter members
- *
- * @param visitableObj
- * @param metadata
- * @throws FilterUnsupportedException,if exception occurs while evaluating
- * filter models.
- */
- public void populateFilterResolvedInfo(DimColumnResolvedFilterInfo visitableObj,
- FilterResolverMetadata metadata) throws FilterUnsupportedException {
- DimColumnFilterInfo resolvedFilterObject = null;
-
- List<String> evaluateResultListFinal;
- try {
- evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString();
- } catch (FilterIllegalMemberException e) {
- throw new FilterUnsupportedException(e);
- }
- resolvedFilterObject = getDirectDictionaryValKeyMemberForFilter(metadata.getColumnExpression(),
- evaluateResultListFinal, metadata.isIncludeFilter());
- if (!metadata.isIncludeFilter() && null != resolvedFilterObject && !resolvedFilterObject
- .getFilterList().contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY)) {
- // Adding default surrogate key of null member inorder to not display the same while
- // displaying the report as per hive compatibility.
- resolvedFilterObject.getFilterList()
- .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
- Collections.sort(resolvedFilterObject.getFilterList());
- }
- visitableObj.setFilterValues(resolvedFilterObject);
- }
-
- private DimColumnFilterInfo getDirectDictionaryValKeyMemberForFilter(
- ColumnExpression columnExpression, List<String> evaluateResultListFinal,
- boolean isIncludeFilter) {
- List<Integer> surrogates = new ArrayList<Integer>(20);
- DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
- .getDirectDictionaryGenerator(columnExpression.getDimension().getDataType());
- // Reading the dictionary value direct
- getSurrogateValuesForDictionary(evaluateResultListFinal, surrogates,
- directDictionaryGenerator);
-
- Collections.sort(surrogates);
- DimColumnFilterInfo columnFilterInfo = null;
- if (surrogates.size() > 0) {
- columnFilterInfo = new DimColumnFilterInfo();
- columnFilterInfo.setIncludeFilter(isIncludeFilter);
- columnFilterInfo.setFilterList(surrogates);
- }
- return columnFilterInfo;
- }
-
- private void getSurrogateValuesForDictionary(List<String> evaluateResultListFinal,
- List<Integer> surrogates, DirectDictionaryGenerator directDictionaryGenerator) {
- String timeFormat = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT);
- for (String filterMember : evaluateResultListFinal) {
- surrogates
- .add(directDictionaryGenerator.generateDirectSurrogateKey(filterMember, timeFormat));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
deleted file mode 100644
index 532af60..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/DictionaryColumnVisitor.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.resolver.resolverinfo.visitor;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.DimColumnFilterInfo;
-import org.apache.carbondata.scan.filter.FilterUtil;
-import org.apache.carbondata.scan.filter.resolver.metadata.FilterResolverMetadata;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-
-public class DictionaryColumnVisitor implements ResolvedFilterInfoVisitorIntf {
-
- /**
- * This Visitor method is used to populate the visitableObj with direct dictionary filter details
- * where the filters values will be resolve using dictionary cache.
- *
- * @param visitableObj
- * @param metadata
- * @throws FilterUnsupportedException,if exception occurs while evaluating
- * filter models.
- * @throws IOException
- * @throws FilterUnsupportedException
- */
- public void populateFilterResolvedInfo(DimColumnResolvedFilterInfo visitableObj,
- FilterResolverMetadata metadata) throws FilterUnsupportedException, IOException {
- DimColumnFilterInfo resolvedFilterObject = null;
- List<String> evaluateResultListFinal;
- try {
- evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString();
- } catch (FilterIllegalMemberException e) {
- throw new FilterUnsupportedException(e);
- }
- resolvedFilterObject = FilterUtil
- .getFilterValues(metadata.getTableIdentifier(), metadata.getColumnExpression(),
- evaluateResultListFinal, metadata.isIncludeFilter());
- if (!metadata.isIncludeFilter() && null != resolvedFilterObject) {
- // Adding default surrogate key of null member inorder to not display the same while
- // displaying the report as per hive compatibility.
- resolvedFilterObject.getFilterList()
- .add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
- Collections.sort(resolvedFilterObject.getFilterList());
- }
- visitableObj.setFilterValues(resolvedFilterObject);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/FilterInfoTypeVisitorFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/FilterInfoTypeVisitorFactory.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/FilterInfoTypeVisitorFactory.java
deleted file mode 100644
index ffb02fc..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/FilterInfoTypeVisitorFactory.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.resolver.resolverinfo.visitor;
-
-import org.apache.carbondata.core.metadata.Encoding;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-
-public class FilterInfoTypeVisitorFactory {
-
- /**
- * This factory method will be used in order to get the visitor instance based on the
- * column expression metadata where filters has been applied.
- *
- * @param columnExpression
- * @return
- */
- public static ResolvedFilterInfoVisitorIntf getResolvedFilterInfoVisitor(
- ColumnExpression columnExpression) {
- if (columnExpression.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
- return new CustomTypeDictionaryVisitor();
- } else if (!columnExpression.getDimension().hasEncoding(Encoding.DICTIONARY)) {
- return new NoDictionaryTypeVisitor();
- } else if (columnExpression.getDimension().hasEncoding(Encoding.DICTIONARY)) {
- return new DictionaryColumnVisitor();
- }
-
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
deleted file mode 100644
index 035fd0d..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/NoDictionaryTypeVisitor.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.resolver.resolverinfo.visitor;
-
-import java.util.List;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.DimColumnFilterInfo;
-import org.apache.carbondata.scan.filter.FilterUtil;
-import org.apache.carbondata.scan.filter.resolver.metadata.FilterResolverMetadata;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-
-public class NoDictionaryTypeVisitor implements ResolvedFilterInfoVisitorIntf {
-
- /**
- * Visitor Method will update the filter related details in visitableObj, For no dictionary
- * type columns the filter members will resolved directly, no need to look up in dictionary
- * since it will not be part of dictionary, directly the actual data can be converted as
- * byte[] and can be set. this type of encoding is effective when the particular column
- * is having very high cardinality.
- *
- * @param visitableObj
- * @param metadata
- * @throws FilterUnsupportedException,if exception occurs while evaluating
- * filter models.
- */
- public void populateFilterResolvedInfo(DimColumnResolvedFilterInfo visitableObj,
- FilterResolverMetadata metadata) throws FilterUnsupportedException {
- DimColumnFilterInfo resolvedFilterObject = null;
- List<String> evaluateResultListFinal;
- try {
- evaluateResultListFinal = metadata.getExpression().evaluate(null).getListAsString();
- // Adding default null member inorder to not display the same while
- // displaying the report as per hive compatibility.
- if (!metadata.isIncludeFilter() && !evaluateResultListFinal
- .contains(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
- evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
- }
- } catch (FilterIllegalMemberException e) {
- throw new FilterUnsupportedException(e);
- }
- resolvedFilterObject = FilterUtil
- .getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, metadata.isIncludeFilter());
- visitableObj.setFilterValues(resolvedFilterObject);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/ResolvedFilterInfoVisitorIntf.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/ResolvedFilterInfoVisitorIntf.java b/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/ResolvedFilterInfoVisitorIntf.java
deleted file mode 100644
index 2c46bde..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/resolver/resolverinfo/visitor/ResolvedFilterInfoVisitorIntf.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter.resolver.resolverinfo.visitor;
-
-import java.io.IOException;
-
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.resolver.metadata.FilterResolverMetadata;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-
-public interface ResolvedFilterInfoVisitorIntf {
-
- /**
- * Visitor pattern is been used in this scenario inorder to populate the
- * dimColResolvedFilterInfo visitable object with filter member values based
- * on the visitor type, currently there 3 types of visitors custom,direct
- * and no dictionary, all types of visitor populate the visitable instance
- * as per its buisness logic which is different for all the visitors.
- *
- * @param visitableObj
- * @param metadata
- * @throws FilterUnsupportedException
- */
- void populateFilterResolvedInfo(DimColumnResolvedFilterInfo visitableObj,
- FilterResolverMetadata metadata) throws FilterUnsupportedException, IOException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/model/CarbonQueryPlan.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/model/CarbonQueryPlan.java b/core/src/main/java/org/apache/carbondata/scan/model/CarbonQueryPlan.java
deleted file mode 100644
index c77e59e..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/model/CarbonQueryPlan.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- *
- */
-package org.apache.carbondata.scan.model;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.scan.expression.Expression;
-
-/**
- * This class contains all the logical information about the query like dimensions,measures,
- * sort order, topN etc..
- */
-public class CarbonQueryPlan implements Serializable {
- /**
- *
- */
- private static final long serialVersionUID = -9036044826928017164L;
-
- /**
- * Database name
- */
- private String databaseName;
-
- /**
- * Table name
- */
- private String tableName;
-
- /**
- * List of dimensions.
- * Ex : select employee_name,department_name,sum(salary) from employee, then employee_name
- * and department_name are dimensions
- * If there is no dimensions asked in query then it would be remained as empty.
- */
- private List<QueryDimension> dimensions =
- new ArrayList<QueryDimension>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
-
- /**
- * List of measures.
- * Ex : select employee_name,department_name,sum(salary) from employee, then sum(salary)
- * would be measure.
- * If there is no dimensions asked in query then it would be remained as empty.
- */
- private List<QueryMeasure> measures =
- new ArrayList<QueryMeasure>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
-
- /**
- * expression
- */
- private Expression expression;
-
- /**
- * queryId
- */
- private String queryId;
-
- /**
- * If it is raw detail query, no need to aggregate in backend. And it returns with dictionary data
- * with out decoding.
- */
- private boolean rawDetailQuery;
-
- /**
- * Constructor created with database name and table name.
- *
- * @param databaseName
- * @param tableName
- */
- public CarbonQueryPlan(String databaseName, String tableName) {
- this.tableName = tableName;
- this.databaseName = databaseName;
- }
-
- /**
- * @return the dimensions
- */
- public List<QueryDimension> getDimensions() {
- return dimensions;
- }
-
- public void addDimension(QueryDimension dimension) {
- this.dimensions.add(dimension);
- }
-
- /**
- * @return the measures
- */
- public List<QueryMeasure> getMeasures() {
- return measures;
- }
-
- public void addMeasure(QueryMeasure measure) {
- this.measures.add(measure);
- }
-
- public Expression getFilterExpression() {
- return expression;
- }
-
- public void setFilterExpression(Expression expression) {
- this.expression = expression;
- }
-
- /**
- * @return the databaseName
- */
- public String getDatabaseName() {
- return databaseName;
- }
-
- /**
- * @return the tableName
- */
- public String getTableName() {
- return tableName;
- }
-
- public String getQueryId() {
- return queryId;
- }
-
- public void setQueryId(String queryId) {
- this.queryId = queryId;
- }
-
- public boolean isRawDetailQuery() {
- return rawDetailQuery;
- }
-
- public void setRawDetailQuery(boolean rawDetailQuery) {
- this.rawDetailQuery = rawDetailQuery;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/model/QueryColumn.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/model/QueryColumn.java b/core/src/main/java/org/apache/carbondata/scan/model/QueryColumn.java
deleted file mode 100644
index d026227..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/model/QueryColumn.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.model;
-
-import java.io.Serializable;
-
-/**
- * query column which will have information about column
- */
-public class QueryColumn implements Serializable {
-
- /**
- * serialVersionUID
- */
- private static final long serialVersionUID = -4222306600480181084L;
-
- /**
- * name of the column
- */
- protected String columnName;
-
- /**
- * query order in which result of the query will be send
- */
- private int queryOrder;
-
- public QueryColumn(String columnName) {
- this.columnName = columnName;
- }
-
- /**
- * @return the columnName
- */
- public String getColumnName() {
- return columnName;
- }
-
- /**
- * @return the queryOrder
- */
- public int getQueryOrder() {
- return queryOrder;
- }
-
- /**
- * @param queryOrder the queryOrder to set
- */
- public void setQueryOrder(int queryOrder) {
- this.queryOrder = queryOrder;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/model/QueryDimension.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/model/QueryDimension.java b/core/src/main/java/org/apache/carbondata/scan/model/QueryDimension.java
deleted file mode 100644
index 9c3002d..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/model/QueryDimension.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.model;
-
-import java.io.Serializable;
-
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-
-/**
- * query plan dimension which will holds the information about the query plan dimension
- * this is done to avoid heavy object serialization
- */
-public class QueryDimension extends QueryColumn implements Serializable {
-
- /**
- * serialVersionUID
- */
- private static final long serialVersionUID = -8492704093776645651L;
- /**
- * actual dimension column
- */
- private transient CarbonDimension dimension;
-
- public QueryDimension(String columnName) {
- super(columnName);
- }
-
- /**
- * @return the dimension
- */
- public CarbonDimension getDimension() {
- return dimension;
- }
-
- /**
- * @param dimension the dimension to set
- */
- public void setDimension(CarbonDimension dimension) {
- this.dimension = dimension;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/model/QueryMeasure.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/model/QueryMeasure.java b/core/src/main/java/org/apache/carbondata/scan/model/QueryMeasure.java
deleted file mode 100644
index f01fa9e..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/model/QueryMeasure.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.model;
-
-import java.io.Serializable;
-
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-
-/**
- * query plan measure, this class will holds the information
- * about measure present in the query, this is done to avoid the serialization
- * of the heavy object
- */
-public class QueryMeasure extends QueryColumn implements Serializable {
-
- /**
- * serialVersionUID
- */
- private static final long serialVersionUID = 1035512411375495414L;
-
- /**
- * actual carbon measure object
- */
- private transient CarbonMeasure measure;
-
- public QueryMeasure(String columnName) {
- super(columnName);
- }
-
- /**
- * @return the measure
- */
- public CarbonMeasure getMeasure() {
- return measure;
- }
-
- /**
- * @param measure the measure to set
- */
- public void setMeasure(CarbonMeasure measure) {
- this.measure = measure;
- }
-
-}
[24/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
new file mode 100644
index 0000000..0b82b53
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
@@ -0,0 +1,338 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.datastore.DataRefNodeFinder;
+import org.apache.carbondata.core.datastore.IndexKey;
+import org.apache.carbondata.core.datastore.block.AbstractIndex;
+import org.apache.carbondata.core.datastore.impl.btree.BTreeDataRefNodeFinder;
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.metadata.Encoding;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.scan.expression.BinaryExpression;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.conditional.BinaryConditionalExpression;
+import org.apache.carbondata.core.scan.expression.conditional.ConditionalExpression;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.resolver.ConditionalFilterResolverImpl;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+import org.apache.carbondata.core.scan.filter.resolver.LogicalFilterResolverImpl;
+import org.apache.carbondata.core.scan.filter.resolver.RowLevelFilterResolverImpl;
+import org.apache.carbondata.core.scan.filter.resolver.RowLevelRangeFilterResolverImpl;
+
+public class FilterExpressionProcessor implements FilterProcessor {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(FilterExpressionProcessor.class.getName());
+
+ /**
+ * Implementation will provide the resolved form of filters based on the
+ * filter expression tree which is been passed in Expression instance.
+ *
+ * @param expressionTree , filter expression tree
+ * @param tableIdentifier ,contains carbon store informations
+ * @return a filter resolver tree
+ */
+ public FilterResolverIntf getFilterResolver(Expression expressionTree,
+ AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException, IOException {
+ if (null != expressionTree && null != tableIdentifier) {
+ return getFilterResolvertree(expressionTree, tableIdentifier);
+ }
+ return null;
+ }
+
+ /**
+ * This API will scan the Segment level all btrees and selects the required
+ * block reference nodes inorder to push the same to executer for applying filters
+ * on the respective data reference node.
+ * Following Algorithm is followed in below API
+ * Step:1 Get the start end key based on the filter tree resolver information
+ * Step:2 Prepare the IndexKeys inorder to scan the tree and get the start and end reference
+ * node(block)
+ * Step:3 Once data reference node ranges retrieved traverse the node within this range
+ * and select the node based on the block min and max value and the filter value.
+ * Step:4 The selected blocks will be send to executers for applying the filters with the help
+ * of Filter executers.
+ *
+ */
+ public List<DataRefNode> getFilterredBlocks(DataRefNode btreeNode,
+ FilterResolverIntf filterResolver, AbstractIndex tableSegment,
+ AbsoluteTableIdentifier tableIdentifier) {
+ // Need to get the current dimension tables
+ List<DataRefNode> listOfDataBlocksToScan = new ArrayList<DataRefNode>();
+ // getting the start and end index key based on filter for hitting the
+ // selected block reference nodes based on filter resolver tree.
+ LOGGER.debug("preparing the start and end key for finding"
+ + "start and end block as per filter resolver");
+ List<IndexKey> listOfStartEndKeys = new ArrayList<IndexKey>(2);
+ FilterUtil.traverseResolverTreeAndGetStartAndEndKey(tableSegment.getSegmentProperties(),
+ filterResolver, listOfStartEndKeys);
+ // reading the first value from list which has start key
+ IndexKey searchStartKey = listOfStartEndKeys.get(0);
+ // reading the last value from list which has end key
+ IndexKey searchEndKey = listOfStartEndKeys.get(1);
+ if (null == searchStartKey && null == searchEndKey) {
+ try {
+ // TODO need to handle for no dictionary dimensions
+ searchStartKey =
+ FilterUtil.prepareDefaultStartIndexKey(tableSegment.getSegmentProperties());
+ // TODO need to handle for no dictionary dimensions
+ searchEndKey = FilterUtil.prepareDefaultEndIndexKey(tableSegment.getSegmentProperties());
+ } catch (KeyGenException e) {
+ return listOfDataBlocksToScan;
+ }
+ }
+
+ LOGGER.debug(
+ "Successfully retrieved the start and end key" + "Dictionary Start Key: " + searchStartKey
+ .getDictionaryKeys() + "No Dictionary Start Key " + searchStartKey.getNoDictionaryKeys()
+ + "Dictionary End Key: " + searchEndKey.getDictionaryKeys() + "No Dictionary End Key "
+ + searchEndKey.getNoDictionaryKeys());
+ long startTimeInMillis = System.currentTimeMillis();
+ DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(
+ tableSegment.getSegmentProperties().getEachDimColumnValueSize());
+ DataRefNode startBlock = blockFinder.findFirstDataBlock(btreeNode, searchStartKey);
+ DataRefNode endBlock = blockFinder.findLastDataBlock(btreeNode, searchEndKey);
+ FilterExecuter filterExecuter =
+ FilterUtil.getFilterExecuterTree(filterResolver, tableSegment.getSegmentProperties(),null);
+ while (startBlock != endBlock) {
+ addBlockBasedOnMinMaxValue(filterExecuter, listOfDataBlocksToScan, startBlock);
+ startBlock = startBlock.getNextDataRefNode();
+ }
+ addBlockBasedOnMinMaxValue(filterExecuter, listOfDataBlocksToScan, endBlock);
+ LOGGER.info("Total Time in retrieving the data reference node" + "after scanning the btree " + (
+ System.currentTimeMillis() - startTimeInMillis)
+ + " Total number of data reference node for executing filter(s) " + listOfDataBlocksToScan
+ .size());
+
+ return listOfDataBlocksToScan;
+ }
+
+ /**
+ * Selects the blocks based on col max and min value.
+ *
+ * @param listOfDataBlocksToScan
+ * @param dataRefNode
+ */
+ private void addBlockBasedOnMinMaxValue(FilterExecuter filterExecuter,
+ List<DataRefNode> listOfDataBlocksToScan, DataRefNode dataRefNode) {
+
+ BitSet bitSet = filterExecuter
+ .isScanRequired(dataRefNode.getColumnsMaxValue(), dataRefNode.getColumnsMinValue());
+ if (!bitSet.isEmpty()) {
+ listOfDataBlocksToScan.add(dataRefNode);
+
+ }
+ }
+
+ /**
+ * API will return a filter resolver instance which will be used by
+ * executers to evaluate or execute the filters.
+ *
+ * @param expressionTree , resolver tree which will hold the resolver tree based on
+ * filter expression.
+ * @return FilterResolverIntf type.
+ */
+ private FilterResolverIntf getFilterResolvertree(Expression expressionTree,
+ AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException, IOException {
+ FilterResolverIntf filterEvaluatorTree =
+ createFilterResolverTree(expressionTree, tableIdentifier);
+ traverseAndResolveTree(filterEvaluatorTree, tableIdentifier);
+ return filterEvaluatorTree;
+ }
+
+ /**
+ * constructing the filter resolver tree based on filter expression.
+ * this method will visit each node of the filter resolver and prepares
+ * the surrogates of the filter members which are involved filter
+ * expression.
+ *
+ * @param filterResolverTree
+ * @param tableIdentifier
+ */
+ private void traverseAndResolveTree(FilterResolverIntf filterResolverTree,
+ AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException, IOException {
+ if (null == filterResolverTree) {
+ return;
+ }
+ traverseAndResolveTree(filterResolverTree.getLeft(), tableIdentifier);
+ filterResolverTree.resolve(tableIdentifier);
+ traverseAndResolveTree(filterResolverTree.getRight(), tableIdentifier);
+ }
+
+ /**
+ * Pattern used : Visitor Pattern
+ * Method will create filter resolver tree based on the filter expression tree,
+ * in this algorithm based on the expression instance the resolvers will created
+ *
+ * @param expressionTree
+ * @param tableIdentifier
+ * @return
+ */
+ private FilterResolverIntf createFilterResolverTree(Expression expressionTree,
+ AbsoluteTableIdentifier tableIdentifier) {
+ ExpressionType filterExpressionType = expressionTree.getFilterExpressionType();
+ BinaryExpression currentExpression = null;
+ switch (filterExpressionType) {
+ case OR:
+ currentExpression = (BinaryExpression) expressionTree;
+ return new LogicalFilterResolverImpl(
+ createFilterResolverTree(currentExpression.getLeft(), tableIdentifier),
+ createFilterResolverTree(currentExpression.getRight(), tableIdentifier),
+ currentExpression);
+ case AND:
+ currentExpression = (BinaryExpression) expressionTree;
+ return new LogicalFilterResolverImpl(
+ createFilterResolverTree(currentExpression.getLeft(), tableIdentifier),
+ createFilterResolverTree(currentExpression.getRight(), tableIdentifier),
+ currentExpression);
+ case EQUALS:
+ case IN:
+ return getFilterResolverBasedOnExpressionType(ExpressionType.EQUALS,
+ ((BinaryConditionalExpression) expressionTree).isNull, expressionTree,
+ tableIdentifier, expressionTree);
+ case GREATERTHAN:
+ case GREATERTHAN_EQUALTO:
+ case LESSTHAN:
+ case LESSTHAN_EQUALTO:
+ return getFilterResolverBasedOnExpressionType(ExpressionType.EQUALS, true, expressionTree,
+ tableIdentifier, expressionTree);
+
+ case NOT_EQUALS:
+ case NOT_IN:
+ return getFilterResolverBasedOnExpressionType(ExpressionType.NOT_EQUALS, false,
+ expressionTree, tableIdentifier, expressionTree);
+ case FALSE:
+ return getFilterResolverBasedOnExpressionType(ExpressionType.FALSE, false,
+ expressionTree, tableIdentifier, expressionTree);
+ default:
+ return getFilterResolverBasedOnExpressionType(ExpressionType.UNKNOWN, false, expressionTree,
+ tableIdentifier, expressionTree);
+ }
+ }
+
+ /**
+ * Factory method which will return the resolver instance based on filter expression
+ * expressions.
+ */
+ private FilterResolverIntf getFilterResolverBasedOnExpressionType(
+ ExpressionType filterExpressionType, boolean isExpressionResolve, Expression expression,
+ AbsoluteTableIdentifier tableIdentifier, Expression expressionTree) {
+ BinaryConditionalExpression currentCondExpression = null;
+ ConditionalExpression condExpression = null;
+ switch (filterExpressionType) {
+ case FALSE:
+ return new RowLevelFilterResolverImpl(expression, false, false, tableIdentifier);
+ case EQUALS:
+ currentCondExpression = (BinaryConditionalExpression) expression;
+ if (currentCondExpression.isSingleDimension()
+ && currentCondExpression.getColumnList().get(0).getCarbonColumn().getDataType()
+ != DataType.ARRAY
+ && currentCondExpression.getColumnList().get(0).getCarbonColumn().getDataType()
+ != DataType.STRUCT) {
+ // getting new dim index.
+ if (!currentCondExpression.getColumnList().get(0).getCarbonColumn()
+ .hasEncoding(Encoding.DICTIONARY) || currentCondExpression.getColumnList().get(0)
+ .getCarbonColumn().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ if (FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getLeft())
+ && FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getRight()) || (
+ FilterUtil.checkIfRightExpressionRequireEvaluation(currentCondExpression.getRight())
+ || FilterUtil
+ .checkIfLeftExpressionRequireEvaluation(currentCondExpression.getLeft()))) {
+ return new RowLevelFilterResolverImpl(expression, isExpressionResolve, true,
+ tableIdentifier);
+ }
+ if (currentCondExpression.getFilterExpressionType() == ExpressionType.GREATERTHAN
+ || currentCondExpression.getFilterExpressionType() == ExpressionType.LESSTHAN
+ || currentCondExpression.getFilterExpressionType()
+ == ExpressionType.GREATERTHAN_EQUALTO
+ || currentCondExpression.getFilterExpressionType()
+ == ExpressionType.LESSTHAN_EQUALTO) {
+ return new RowLevelRangeFilterResolverImpl(expression, isExpressionResolve, true,
+ tableIdentifier);
+ }
+ }
+ return new ConditionalFilterResolverImpl(expression, isExpressionResolve, true);
+
+ }
+ break;
+ case NOT_EQUALS:
+ currentCondExpression = (BinaryConditionalExpression) expression;
+ if (currentCondExpression.isSingleDimension()
+ && currentCondExpression.getColumnList().get(0).getCarbonColumn().getDataType()
+ != DataType.ARRAY
+ && currentCondExpression.getColumnList().get(0).getCarbonColumn().getDataType()
+ != DataType.STRUCT) {
+ if (!currentCondExpression.getColumnList().get(0).getCarbonColumn()
+ .hasEncoding(Encoding.DICTIONARY) || currentCondExpression.getColumnList().get(0)
+ .getCarbonColumn().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ if (FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getLeft())
+ && FilterUtil.checkIfExpressionContainsColumn(currentCondExpression.getRight()) || (
+ FilterUtil.checkIfRightExpressionRequireEvaluation(currentCondExpression.getRight())
+ || FilterUtil
+ .checkIfLeftExpressionRequireEvaluation(currentCondExpression.getLeft()))) {
+ return new RowLevelFilterResolverImpl(expression, isExpressionResolve, false,
+ tableIdentifier);
+ }
+ if (expressionTree.getFilterExpressionType() == ExpressionType.GREATERTHAN
+ || expressionTree.getFilterExpressionType() == ExpressionType.LESSTHAN
+ || expressionTree.getFilterExpressionType() == ExpressionType.GREATERTHAN_EQUALTO
+ || expressionTree.getFilterExpressionType() == ExpressionType.LESSTHAN_EQUALTO) {
+
+ return new RowLevelRangeFilterResolverImpl(expression, isExpressionResolve, false,
+ tableIdentifier);
+ }
+
+ return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false);
+ }
+ return new ConditionalFilterResolverImpl(expression, isExpressionResolve, false);
+ }
+ break;
+ default:
+ if(expression instanceof ConditionalExpression) {
+ condExpression = (ConditionalExpression) expression;
+ if (condExpression.isSingleDimension()
+ && condExpression.getColumnList().get(0).getCarbonColumn().getDataType()
+ != DataType.ARRAY
+ && condExpression.getColumnList().get(0).getCarbonColumn().getDataType()
+ != DataType.STRUCT) {
+ condExpression = (ConditionalExpression) expression;
+ if (condExpression.getColumnList().get(0).getCarbonColumn()
+ .hasEncoding(Encoding.DICTIONARY) && !condExpression.getColumnList().get(0)
+ .getCarbonColumn().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ return new ConditionalFilterResolverImpl(expression, true, true);
+ }
+ }
+ }
+ }
+ return new RowLevelFilterResolverImpl(expression, false, false, tableIdentifier);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java
new file mode 100644
index 0000000..4199429
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java
@@ -0,0 +1,57 @@
+package org.apache.carbondata.core.scan.filter;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.datastore.block.AbstractIndex;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+
+public interface FilterProcessor {
+
+ /**
+ * API will provide the resolved form of filters based on the filter
+ * expression tree which is been passed.
+ *
+ * @param expressionTree , filter expression tree
+ * @param tableIdentifier ,contains carbon store informations.
+ * @return
+ * @throws FilterUnsupportedException
+ */
+ FilterResolverIntf getFilterResolver(Expression expressionTree,
+ AbsoluteTableIdentifier tableIdentifier) throws FilterUnsupportedException, IOException;
+
+ /**
+ * This API is exposed inorder to get the required block reference node
+ * based on the filter.The block list will be send to the executer tasks inorder
+ * to apply filters.
+ *
+ * @param filterResolver DataBlock list with resolved filters
+ * @return list of DataRefNode.
+ */
+ List<DataRefNode> getFilterredBlocks(DataRefNode dataRefNode, FilterResolverIntf filterResolver,
+ AbstractIndex segmentIndexBuilder, AbsoluteTableIdentifier tableIdentifier);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
new file mode 100644
index 0000000..ab0c7e8
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -0,0 +1,1353 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.filter;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.cache.Cache;
+import org.apache.carbondata.core.cache.CacheProvider;
+import org.apache.carbondata.core.cache.CacheType;
+import org.apache.carbondata.core.cache.dictionary.Dictionary;
+import org.apache.carbondata.core.cache.dictionary.DictionaryChunksWrapper;
+import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
+import org.apache.carbondata.core.cache.dictionary.ForwardDictionary;
+import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.datastore.IndexKey;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.metadata.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.keygenerator.KeyGenerator;
+import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.LiteralExpression;
+import org.apache.carbondata.core.scan.expression.conditional.ListExpression;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.executer.AndFilterExecuterImpl;
+import org.apache.carbondata.core.scan.filter.executer.DimColumnExecuterFilterInfo;
+import org.apache.carbondata.core.scan.filter.executer.ExcludeColGroupFilterExecuterImpl;
+import org.apache.carbondata.core.scan.filter.executer.ExcludeFilterExecuterImpl;
+import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
+import org.apache.carbondata.core.scan.filter.executer.IncludeColGroupFilterExecuterImpl;
+import org.apache.carbondata.core.scan.filter.executer.IncludeFilterExecuterImpl;
+import org.apache.carbondata.core.scan.filter.executer.OrFilterExecuterImpl;
+import org.apache.carbondata.core.scan.filter.executer.RestructureFilterExecuterImpl;
+import org.apache.carbondata.core.scan.filter.executer.RowLevelFilterExecuterImpl;
+import org.apache.carbondata.core.scan.filter.executer.RowLevelRangeTypeExecuterFacory;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType;
+import org.apache.carbondata.core.scan.filter.intf.RowImpl;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+import org.apache.carbondata.core.scan.filter.resolver.RowLevelFilterResolverImpl;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+
+public final class FilterUtil {
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(FilterUtil.class.getName());
+
+ private FilterUtil() {
+
+ }
+
+ /**
+ * Pattern used : Visitor Pattern
+ * Method will create filter executer tree based on the filter resolved tree,
+ * in this algorithm based on the resolver instance the executers will be visited
+ * and the resolved surrogates will be converted to keys
+ *
+ * @param filterExpressionResolverTree
+ * @param segmentProperties
+ * @return FilterExecuter instance
+ */
+ private static FilterExecuter createFilterExecuterTree(
+ FilterResolverIntf filterExpressionResolverTree, SegmentProperties segmentProperties,
+ Map<Integer, GenericQueryType> complexDimensionInfoMap) {
+ FilterExecuterType filterExecuterType = filterExpressionResolverTree.getFilterExecuterType();
+ if (null != filterExecuterType) {
+ switch (filterExecuterType) {
+ case INCLUDE:
+ return getIncludeFilterExecuter(
+ filterExpressionResolverTree.getDimColResolvedFilterInfo(), segmentProperties);
+ case EXCLUDE:
+ return getExcludeFilterExecuter(
+ filterExpressionResolverTree.getDimColResolvedFilterInfo(), segmentProperties);
+ case OR:
+ return new OrFilterExecuterImpl(
+ createFilterExecuterTree(filterExpressionResolverTree.getLeft(), segmentProperties,
+ complexDimensionInfoMap),
+ createFilterExecuterTree(filterExpressionResolverTree.getRight(), segmentProperties,
+ complexDimensionInfoMap));
+ case AND:
+ return new AndFilterExecuterImpl(
+ createFilterExecuterTree(filterExpressionResolverTree.getLeft(), segmentProperties,
+ complexDimensionInfoMap),
+ createFilterExecuterTree(filterExpressionResolverTree.getRight(), segmentProperties,
+ complexDimensionInfoMap));
+ case RESTRUCTURE:
+ return new RestructureFilterExecuterImpl(
+ filterExpressionResolverTree.getDimColResolvedFilterInfo(),
+ segmentProperties);
+ case ROWLEVEL_LESSTHAN:
+ case ROWLEVEL_LESSTHAN_EQUALTO:
+ case ROWLEVEL_GREATERTHAN_EQUALTO:
+ case ROWLEVEL_GREATERTHAN:
+ return RowLevelRangeTypeExecuterFacory
+ .getRowLevelRangeTypeExecuter(filterExecuterType, filterExpressionResolverTree,
+ segmentProperties);
+ case ROWLEVEL:
+ default:
+ return new RowLevelFilterExecuterImpl(
+ ((RowLevelFilterResolverImpl) filterExpressionResolverTree)
+ .getDimColEvaluatorInfoList(),
+ ((RowLevelFilterResolverImpl) filterExpressionResolverTree)
+ .getMsrColEvalutorInfoList(),
+ ((RowLevelFilterResolverImpl) filterExpressionResolverTree).getFilterExpresion(),
+ ((RowLevelFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(),
+ segmentProperties, complexDimensionInfoMap);
+
+ }
+ }
+ return new RowLevelFilterExecuterImpl(
+ ((RowLevelFilterResolverImpl) filterExpressionResolverTree).getDimColEvaluatorInfoList(),
+ ((RowLevelFilterResolverImpl) filterExpressionResolverTree).getMsrColEvalutorInfoList(),
+ ((RowLevelFilterResolverImpl) filterExpressionResolverTree).getFilterExpresion(),
+ ((RowLevelFilterResolverImpl) filterExpressionResolverTree).getTableIdentifier(),
+ segmentProperties, complexDimensionInfoMap);
+
+ }
+
+ /**
+ * It gives filter executer based on columnar or column group
+ *
+ * @param dimColResolvedFilterInfo
+ * @param segmentProperties
+ * @return
+ */
+ private static FilterExecuter getIncludeFilterExecuter(
+ DimColumnResolvedFilterInfo dimColResolvedFilterInfo, SegmentProperties segmentProperties) {
+
+ if (dimColResolvedFilterInfo.getDimension().isColumnar()) {
+ return new IncludeFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
+ } else {
+ return new IncludeColGroupFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
+ }
+ }
+
+ /**
+ * It gives filter executer based on columnar or column group
+ *
+ * @param dimColResolvedFilterInfo
+ * @param segmentProperties
+ * @return
+ */
+ private static FilterExecuter getExcludeFilterExecuter(
+ DimColumnResolvedFilterInfo dimColResolvedFilterInfo, SegmentProperties segmentProperties) {
+
+ if (dimColResolvedFilterInfo.getDimension().isColumnar()) {
+ return new ExcludeFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
+ } else {
+ return new ExcludeColGroupFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
+ }
+ }
+
+ /**
+ * This method will check if a given expression contains a column expression
+ * recursively.
+ *
+ * @return
+ */
+ public static boolean checkIfExpressionContainsColumn(Expression expression) {
+ if (expression instanceof ColumnExpression) {
+ return true;
+ }
+ for (Expression child : expression.getChildren()) {
+ if (checkIfExpressionContainsColumn(child)) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ /**
+ * This method will check if a given expression contains a column expression
+ * recursively.
+ *
+ * @return
+ */
+ public static boolean checkIfLeftExpressionRequireEvaluation(Expression expression) {
+ if (expression.getFilterExpressionType() == ExpressionType.UNKNOWN
+ || !(expression instanceof ColumnExpression)) {
+ return true;
+ }
+ for (Expression child : expression.getChildren()) {
+ if (checkIfLeftExpressionRequireEvaluation(child)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * This method will check if a given literal expression is not a timestamp datatype
+ * recursively.
+ *
+ * @return
+ */
+ public static boolean checkIfDataTypeNotTimeStamp(Expression expression) {
+ if (expression.getFilterExpressionType() == ExpressionType.LITERAL) {
+ DataType dataType = ((LiteralExpression) expression).getLiteralExpDataType();
+ if (!(dataType == DataType.TIMESTAMP || dataType == DataType.DATE)) {
+ return true;
+ }
+ }
+ for (Expression child : expression.getChildren()) {
+ if (checkIfDataTypeNotTimeStamp(child)) {
+ return true;
+ }
+ }
+ return false;
+ }
+ /**
+ * This method will check if a given expression contains a column expression
+ * recursively.
+ *
+ * @return
+ */
+ public static boolean checkIfRightExpressionRequireEvaluation(Expression expression) {
+ if (expression.getFilterExpressionType() == ExpressionType.UNKNOWN
+ || !(expression instanceof LiteralExpression) && !(expression instanceof ListExpression)) {
+ return true;
+ }
+ for (Expression child : expression.getChildren()) {
+ if (checkIfRightExpressionRequireEvaluation(child)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * method will get the masked keys based on the keys generated from surrogates.
+ *
+ * @param ranges
+ * @param key
+ * @return byte[]
+ */
+ private static byte[] getMaskedKey(int[] ranges, byte[] key) {
+ byte[] maskkey = new byte[ranges.length];
+
+ for (int i = 0; i < maskkey.length; i++) {
+ maskkey[i] = key[ranges[i]];
+ }
+ return maskkey;
+ }
+
+ /**
+ * This method will return the ranges for the masked Bytes based on the key
+ * Generator.
+ *
+ * @param queryDimensionsOrdinal
+ * @param generator
+ * @return
+ */
+ private static int[] getRangesForMaskedByte(int queryDimensionsOrdinal, KeyGenerator generator) {
+ Set<Integer> integers = new TreeSet<Integer>();
+ int[] range = generator.getKeyByteOffsets(queryDimensionsOrdinal);
+ for (int j = range[0]; j <= range[1]; j++) {
+ integers.add(j);
+ }
+
+ int[] byteIndexs = new int[integers.size()];
+ int j = 0;
+ for (Iterator<Integer> iterator = integers.iterator(); iterator.hasNext(); ) {
+ Integer integer = iterator.next();
+ byteIndexs[j++] = integer;
+ }
+ return byteIndexs;
+ }
+
+ /**
+ * This method will get the no dictionary data based on filters and same
+ * will be in DimColumnFilterInfo
+ *
+ * @param evaluateResultListFinal
+ * @param isIncludeFilter
+ * @return DimColumnFilterInfo
+ */
+ public static DimColumnFilterInfo getNoDictionaryValKeyMemberForFilter(
+ List<String> evaluateResultListFinal, boolean isIncludeFilter) {
+ List<byte[]> filterValuesList = new ArrayList<byte[]>(20);
+ for (String result : evaluateResultListFinal) {
+ filterValuesList.add(result.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
+ }
+
+ Comparator<byte[]> filterNoDictValueComaparator = new Comparator<byte[]>() {
+
+ @Override public int compare(byte[] filterMember1, byte[] filterMember2) {
+ // TODO Auto-generated method stub
+ return ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterMember1, filterMember2);
+ }
+
+ };
+ Collections.sort(filterValuesList, filterNoDictValueComaparator);
+ DimColumnFilterInfo columnFilterInfo = null;
+ if (filterValuesList.size() > 0) {
+ columnFilterInfo = new DimColumnFilterInfo();
+ columnFilterInfo.setIncludeFilter(isIncludeFilter);
+ columnFilterInfo.setFilterListForNoDictionaryCols(filterValuesList);
+
+ }
+ return columnFilterInfo;
+ }
+
+ /**
+ * Method will prepare the dimfilterinfo instance by resolving the filter
+ * expression value to its respective surrogates.
+ *
+ * @param tableIdentifier
+ * @param columnExpression
+ * @param evaluateResultList
+ * @param isIncludeFilter
+ * @return
+ * @throws IOException
+ */
+ public static DimColumnFilterInfo getFilterValues(AbsoluteTableIdentifier tableIdentifier,
+ ColumnExpression columnExpression, List<String> evaluateResultList, boolean isIncludeFilter)
+ throws IOException {
+ Dictionary forwardDictionary = null;
+ try {
+ // Reading the dictionary value from cache.
+ forwardDictionary =
+ getForwardDictionaryCache(tableIdentifier, columnExpression.getDimension());
+ return getFilterValues(columnExpression, evaluateResultList, forwardDictionary,
+ isIncludeFilter);
+ } finally {
+ CarbonUtil.clearDictionaryCache(forwardDictionary);
+ }
+ }
+
+ /**
+ * Method will prepare the dimfilterinfo instance by resolving the filter
+ * expression value to its respective surrogates.
+ *
+ * @param columnExpression
+ * @param evaluateResultList
+ * @param forwardDictionary
+ * @param isIncludeFilter
+ * @return
+ */
+ private static DimColumnFilterInfo getFilterValues(ColumnExpression columnExpression,
+ List<String> evaluateResultList, Dictionary forwardDictionary, boolean isIncludeFilter) {
+ sortFilterModelMembers(columnExpression, evaluateResultList);
+ List<Integer> surrogates =
+ new ArrayList<Integer>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ // Reading the dictionary value from cache.
+ getDictionaryValue(evaluateResultList, forwardDictionary, surrogates);
+ Collections.sort(surrogates);
+ DimColumnFilterInfo columnFilterInfo = null;
+ if (surrogates.size() > 0) {
+ columnFilterInfo = new DimColumnFilterInfo();
+ columnFilterInfo.setIncludeFilter(isIncludeFilter);
+ columnFilterInfo.setFilterList(surrogates);
+ }
+ return columnFilterInfo;
+ }
+
+ /**
+ * This API will get the Dictionary value for the respective filter member
+ * string.
+ *
+ * @param evaluateResultList filter value
+ * @param surrogates
+ */
+ private static void getDictionaryValue(List<String> evaluateResultList,
+ Dictionary forwardDictionary, List<Integer> surrogates) {
+ ((ForwardDictionary) forwardDictionary)
+ .getSurrogateKeyByIncrementalSearch(evaluateResultList, surrogates);
+ }
+
+ /**
+ * This method will get all the members of column from the forward dictionary
+ * cache, this method will be basically used in row level filter resolver.
+ *
+ * @param tableIdentifier
+ * @param expression
+ * @param columnExpression
+ * @param isIncludeFilter
+ * @return DimColumnFilterInfo
+ * @throws FilterUnsupportedException
+ * @throws IOException
+ */
+ public static DimColumnFilterInfo getFilterListForAllValues(
+ AbsoluteTableIdentifier tableIdentifier, Expression expression,
+ final ColumnExpression columnExpression, boolean isIncludeFilter)
+ throws IOException, FilterUnsupportedException {
+ Dictionary forwardDictionary = null;
+ List<String> evaluateResultListFinal = new ArrayList<String>(20);
+ DictionaryChunksWrapper dictionaryWrapper = null;
+ try {
+ forwardDictionary =
+ getForwardDictionaryCache(tableIdentifier, columnExpression.getDimension());
+ dictionaryWrapper = forwardDictionary.getDictionaryChunks();
+ while (dictionaryWrapper.hasNext()) {
+ byte[] columnVal = dictionaryWrapper.next();
+ try {
+ RowIntf row = new RowImpl();
+ String stringValue =
+ new String(columnVal, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+ if (stringValue.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
+ stringValue = null;
+ }
+ row.setValues(new Object[] { DataTypeUtil.getDataBasedOnDataType(stringValue,
+ columnExpression.getCarbonColumn().getDataType()) });
+ Boolean rslt = expression.evaluate(row).getBoolean();
+ if (null != rslt && rslt == isIncludeFilter) {
+ if (null == stringValue) {
+ evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
+ } else {
+ evaluateResultListFinal.add(stringValue);
+ }
+ }
+ } catch (FilterIllegalMemberException e) {
+ LOGGER.debug(e.getMessage());
+ }
+ }
+ return getFilterValues(columnExpression, evaluateResultListFinal, forwardDictionary,
+ isIncludeFilter);
+ } finally {
+ CarbonUtil.clearDictionaryCache(forwardDictionary);
+ }
+ }
+
+ private static void sortFilterModelMembers(final ColumnExpression columnExpression,
+ List<String> evaluateResultListFinal) {
+ Comparator<String> filterActualValueComaparator = new Comparator<String>() {
+
+ @Override public int compare(String filterMember1, String filterMember2) {
+ return compareFilterMembersBasedOnActualDataType(filterMember1, filterMember2,
+ columnExpression.getDataType());
+ }
+
+ };
+ Collections.sort(evaluateResultListFinal, filterActualValueComaparator);
+ }
+
+ /**
+ * Method will prepare the dimfilterinfo instance by resolving the filter
+ * expression value to its respective surrogates in the scenario of restructure.
+ *
+ * @param expression
+ * @param columnExpression
+ * @param defaultValues
+ * @param defaultSurrogate
+ * @return
+ * @throws FilterUnsupportedException
+ */
+ public static DimColumnFilterInfo getFilterListForRS(Expression expression,
+ ColumnExpression columnExpression, String defaultValues, int defaultSurrogate)
+ throws FilterUnsupportedException {
+ List<Integer> filterValuesList = new ArrayList<Integer>(20);
+ DimColumnFilterInfo columnFilterInfo = null;
+ // List<byte[]> filterValuesList = new ArrayList<byte[]>(20);
+ List<String> evaluateResultListFinal = new ArrayList<String>(20);
+ // KeyGenerator keyGenerator =
+ // KeyGeneratorFactory.getKeyGenerator(new int[] { defaultSurrogate });
+ try {
+ List<ExpressionResult> evaluateResultList = expression.evaluate(null).getList();
+ for (ExpressionResult result : evaluateResultList) {
+ if (result.getString() == null) {
+ evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
+ continue;
+ }
+ evaluateResultListFinal.add(result.getString());
+ }
+
+ for (int i = 0; i < evaluateResultListFinal.size(); i++) {
+ if (evaluateResultListFinal.get(i).equals(defaultValues)) {
+ filterValuesList.add(defaultSurrogate);
+ break;
+ }
+ }
+ if (filterValuesList.size() > 0) {
+ columnFilterInfo = new DimColumnFilterInfo();
+ columnFilterInfo.setFilterList(filterValuesList);
+ }
+ } catch (FilterIllegalMemberException e) {
+ LOGGER.audit(e.getMessage());
+ }
+ return columnFilterInfo;
+ }
+
+ /**
+ * This method will get the member based on filter expression evaluation from the
+ * forward dictionary cache, this method will be basically used in restructure.
+ *
+ * @param expression
+ * @param columnExpression
+ * @param defaultValues
+ * @param defaultSurrogate
+ * @param isIncludeFilter
+ * @return
+ * @throws FilterUnsupportedException
+ */
+ public static DimColumnFilterInfo getFilterListForAllMembersRS(Expression expression,
+ ColumnExpression columnExpression, String defaultValues, int defaultSurrogate,
+ boolean isIncludeFilter) throws FilterUnsupportedException {
+ List<Integer> filterValuesList = new ArrayList<Integer>(20);
+ List<String> evaluateResultListFinal = new ArrayList<String>(20);
+ DimColumnFilterInfo columnFilterInfo = null;
+
+ // KeyGenerator keyGenerator =
+ // KeyGeneratorFactory.getKeyGenerator(new int[] { defaultSurrogate });
+ try {
+ RowIntf row = new RowImpl();
+ if (defaultValues.equals(CarbonCommonConstants.MEMBER_DEFAULT_VAL)) {
+ defaultValues = null;
+ }
+ row.setValues(new Object[] { DataTypeUtil.getDataBasedOnDataType(defaultValues,
+ columnExpression.getCarbonColumn().getDataType()) });
+ Boolean rslt = expression.evaluate(row).getBoolean();
+ if (null != rslt && rslt == isIncludeFilter) {
+ if (null == defaultValues) {
+ evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
+ } else {
+ evaluateResultListFinal.add(defaultValues);
+ }
+ }
+ } catch (FilterIllegalMemberException e) {
+ LOGGER.audit(e.getMessage());
+ }
+
+ if (null == defaultValues) {
+ defaultValues = CarbonCommonConstants.MEMBER_DEFAULT_VAL;
+ }
+ columnFilterInfo = new DimColumnFilterInfo();
+ for (int i = 0; i < evaluateResultListFinal.size(); i++) {
+ if (evaluateResultListFinal.get(i).equals(defaultValues)) {
+ filterValuesList.add(defaultSurrogate);
+ break;
+ }
+ }
+ columnFilterInfo.setFilterList(filterValuesList);
+ return columnFilterInfo;
+ }
+
+ /**
+ * Below method will be used to covert the filter surrogate keys
+ * to mdkey
+ *
+ * @param dimColumnFilterInfo
+ * @param carbonDimension
+ * @param segmentProperties
+ * @return
+ */
+ public static byte[][] getKeyArray(DimColumnFilterInfo dimColumnFilterInfo,
+ CarbonDimension carbonDimension, SegmentProperties segmentProperties) {
+ if (!carbonDimension.hasEncoding(Encoding.DICTIONARY)) {
+ return dimColumnFilterInfo.getNoDictionaryFilterValuesList()
+ .toArray((new byte[dimColumnFilterInfo.getNoDictionaryFilterValuesList().size()][]));
+ }
+ KeyGenerator blockLevelKeyGenerator = segmentProperties.getDimensionKeyGenerator();
+ int[] dimColumnsCardinality = segmentProperties.getDimColumnsCardinality();
+ int[] keys = new int[blockLevelKeyGenerator.getDimCount()];
+ List<byte[]> filterValuesList = new ArrayList<byte[]>(20);
+ Arrays.fill(keys, 0);
+ int[] rangesForMaskedByte =
+ getRangesForMaskedByte((carbonDimension.getKeyOrdinal()), blockLevelKeyGenerator);
+ if (null != dimColumnFilterInfo) {
+ for (Integer surrogate : dimColumnFilterInfo.getFilterList()) {
+ try {
+ if (surrogate <= dimColumnsCardinality[carbonDimension.getKeyOrdinal()]) {
+ keys[carbonDimension.getKeyOrdinal()] = surrogate;
+ filterValuesList
+ .add(getMaskedKey(rangesForMaskedByte, blockLevelKeyGenerator.generateKey(keys)));
+ } else {
+ break;
+ }
+ } catch (KeyGenException e) {
+ LOGGER.error(e.getMessage());
+ }
+ }
+
+ }
+ return filterValuesList.toArray(new byte[filterValuesList.size()][]);
+
+ }
+
+ /**
+ * The method is used to get the single dictionary key's mask key
+ *
+ * @param surrogate
+ * @param carbonDimension
+ * @param blockLevelKeyGenerator
+ * @return
+ */
+ public static byte[] getMaskKey(int surrogate, CarbonDimension carbonDimension,
+ KeyGenerator blockLevelKeyGenerator) {
+
+ int[] keys = new int[blockLevelKeyGenerator.getDimCount()];
+ byte[] maskedKey = null;
+ Arrays.fill(keys, 0);
+ int[] rangesForMaskedByte =
+ getRangesForMaskedByte((carbonDimension.getKeyOrdinal()), blockLevelKeyGenerator);
+ try {
+ keys[carbonDimension.getKeyOrdinal()] = surrogate;
+ maskedKey = getMaskedKey(rangesForMaskedByte, blockLevelKeyGenerator.generateKey(keys));
+ } catch (KeyGenException e) {
+ LOGGER.error(e.getMessage());
+ }
+ return maskedKey;
+ }
+
+ /**
+ * Method will return the start key based on KeyGenerator for the respective
+ * filter resolved instance.
+ *
+ * @param dimensionFilter
+ * @param startKey
+ * @param startKeyList
+ * @return long[] start key
+ */
+ public static void getStartKey(Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter,
+ long[] startKey, List<long[]> startKeyList) {
+ for(int i = 0; i < startKey.length; i++) {
+ // The min surrogate key is 1, set it as the init value for starkey of each column level
+ startKey[i] = 1;
+ }
+ getStartKeyWithFilter(dimensionFilter, startKey, startKeyList);
+ }
+
+ /**
+ * Algorithm for getting the start key for a filter
+ * step 1: Iterate through each dimension and verify whether its not an exclude filter.
+ * step 2: Intialize start key with the first filter member value present in each filter model
+ * for the respective dimensions.
+ * step 3: since its a no dictionary start key there will only actual value so compare
+ * the first filter model value with respect to the dimension data type.
+ * step 4: The least value will be considered as the start key of dimension by comparing all
+ * its filter model.
+ * step 5: create a byte array of start key which comprises of least filter member value of
+ * all dimension and the indexes which will help to read the respective filter value.
+ *
+ * @param dimColResolvedFilterInfo
+ * @param setOfStartKeyByteArray
+ * @return
+ */
+ public static void getStartKeyForNoDictionaryDimension(
+ DimColumnResolvedFilterInfo dimColResolvedFilterInfo,
+ SortedMap<Integer, byte[]> setOfStartKeyByteArray) {
+ Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter =
+ dimColResolvedFilterInfo.getDimensionResolvedFilterInstance();
+ // step 1
+ for (Map.Entry<CarbonDimension, List<DimColumnFilterInfo>> entry : dimensionFilter.entrySet()) {
+ if (!entry.getKey().hasEncoding(Encoding.DICTIONARY)) {
+ List<DimColumnFilterInfo> listOfDimColFilterInfo = entry.getValue();
+ if (null == listOfDimColFilterInfo) {
+ continue;
+ }
+ boolean isExcludePresent = false;
+ for (DimColumnFilterInfo info : listOfDimColFilterInfo) {
+ if (!info.isIncludeFilter()) {
+ isExcludePresent = true;
+ }
+ }
+ if (isExcludePresent) {
+ continue;
+ }
+ // step 2
+ byte[] noDictionaryStartKey =
+ listOfDimColFilterInfo.get(0).getNoDictionaryFilterValuesList().get(0);
+ if (setOfStartKeyByteArray.isEmpty()) {
+ setOfStartKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryStartKey);
+ } else if (null == setOfStartKeyByteArray.get(entry.getKey().getOrdinal())) {
+ setOfStartKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryStartKey);
+
+ } else if (ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(setOfStartKeyByteArray.get(entry.getKey().getOrdinal()),
+ noDictionaryStartKey) > 0) {
+ setOfStartKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryStartKey);
+ }
+ }
+ }
+ }
+
+ /**
+ * Algorithm for getting the end key for a filter
+ * step 1: Iterate through each dimension and verify whether its not an exclude filter.
+ * step 2: Initialize end key with the last filter member value present in each filter model
+ * for the respective dimensions.(Already filter models are sorted)
+ * step 3: since its a no dictionary end key there will only actual value so compare
+ * the last filter model value with respect to the dimension data type.
+ * step 4: The highest value will be considered as the end key of dimension by comparing all
+ * its filter model.
+ * step 5: create a byte array of end key which comprises of highest filter member value of
+ * all dimension and the indexes which will help to read the respective filter value.
+ *
+ * @param dimColResolvedFilterInfo
+ * @param setOfEndKeyByteArray
+ * @return end key array
+ */
+ public static void getEndKeyForNoDictionaryDimension(
+ DimColumnResolvedFilterInfo dimColResolvedFilterInfo,
+ SortedMap<Integer, byte[]> setOfEndKeyByteArray) {
+
+ Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter =
+ dimColResolvedFilterInfo.getDimensionResolvedFilterInstance();
+ // step 1
+ for (Map.Entry<CarbonDimension, List<DimColumnFilterInfo>> entry : dimensionFilter.entrySet()) {
+ if (!entry.getKey().hasEncoding(Encoding.DICTIONARY)) {
+ List<DimColumnFilterInfo> listOfDimColFilterInfo = entry.getValue();
+ if (null == listOfDimColFilterInfo) {
+ continue;
+ }
+ boolean isExcludePresent = false;
+ for (DimColumnFilterInfo info : listOfDimColFilterInfo) {
+ if (!info.isIncludeFilter()) {
+ isExcludePresent = true;
+ }
+ }
+ if (isExcludePresent) {
+ continue;
+ }
+ // step 2
+ byte[] noDictionaryEndKey = listOfDimColFilterInfo.get(0).getNoDictionaryFilterValuesList()
+ .get(listOfDimColFilterInfo.get(0).getNoDictionaryFilterValuesList().size() - 1);
+ if (setOfEndKeyByteArray.isEmpty()) {
+ setOfEndKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryEndKey);
+ } else if (null == setOfEndKeyByteArray.get(entry.getKey().getOrdinal())) {
+ setOfEndKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryEndKey);
+
+ } else if (ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(setOfEndKeyByteArray.get(entry.getKey().getOrdinal()), noDictionaryEndKey)
+ < 0) {
+ setOfEndKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryEndKey);
+ }
+
+ }
+ }
+ }
+
+ /**
+ * Method will pack all the byte[] to a single byte[] value by appending the
+ * indexes of the byte[] value which needs to be read. this method will be mailny used
+ * in case of no dictionary dimension processing for filters.
+ *
+ * @param noDictionaryValKeyList
+ * @return packed key with its indexes added in starting and its actual values.
+ */
+ private static byte[] getKeyWithIndexesAndValues(List<byte[]> noDictionaryValKeyList) {
+ ByteBuffer[] buffArr = new ByteBuffer[noDictionaryValKeyList.size()];
+ int index = 0;
+ for (byte[] singleColVal : noDictionaryValKeyList) {
+ buffArr[index] = ByteBuffer.allocate(singleColVal.length);
+ buffArr[index].put(singleColVal);
+ buffArr[index++].rewind();
+ }
+ // byteBufer.
+ return CarbonUtil.packByteBufferIntoSingleByteArray(buffArr);
+
+ }
+
+ /**
+ * This method will fill the start key array with the surrogate key present
+ * in filterinfo instance.
+ *
+ * @param dimensionFilter
+ * @param startKey
+ */
+ private static void getStartKeyWithFilter(
+ Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter, long[] startKey,
+ List<long[]> startKeyList) {
+ for (Map.Entry<CarbonDimension, List<DimColumnFilterInfo>> entry : dimensionFilter.entrySet()) {
+ List<DimColumnFilterInfo> values = entry.getValue();
+ if (null == values || !entry.getKey().hasEncoding(Encoding.DICTIONARY)) {
+ continue;
+ }
+ boolean isExcludePresent = false;
+ for (DimColumnFilterInfo info : values) {
+ if (!info.isIncludeFilter()) {
+ isExcludePresent = true;
+ }
+ }
+ if (isExcludePresent) {
+ continue;
+ }
+ for (DimColumnFilterInfo info : values) {
+ if (startKey[entry.getKey().getKeyOrdinal()] < info.getFilterList().get(0)) {
+ startKey[entry.getKey().getKeyOrdinal()] = info.getFilterList().get(0);
+ }
+ }
+ long[] newStartKey = new long[startKey.length];
+ System.arraycopy(startKey, 0, newStartKey, 0, startKey.length);
+ startKeyList.add(newStartKey);
+ }
+ }
+
+ public static void getEndKey(Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter,
+ long[] endKey, SegmentProperties segmentProperties,
+ List<long[]> endKeyList) {
+
+ List<CarbonDimension> updatedDimListBasedOnKeyGenerator =
+ getCarbonDimsMappedToKeyGenerator(segmentProperties.getDimensions());
+ for (int i = 0; i < endKey.length; i++) {
+ endKey[i] = getMaxValue(updatedDimListBasedOnKeyGenerator.get(i),
+ segmentProperties.getDimColumnsCardinality());
+ }
+ getEndKeyWithFilter(dimensionFilter, endKey, endKeyList);
+
+ }
+
+ private static List<CarbonDimension> getCarbonDimsMappedToKeyGenerator(
+ List<CarbonDimension> carbonDimensions) {
+ List<CarbonDimension> listOfCarbonDimPartOfKeyGen =
+ new ArrayList<CarbonDimension>(carbonDimensions.size());
+ for (CarbonDimension carbonDim : carbonDimensions) {
+ if (CarbonUtil.hasEncoding(carbonDim.getEncoder(), Encoding.DICTIONARY) || CarbonUtil
+ .hasEncoding(carbonDim.getEncoder(), Encoding.DIRECT_DICTIONARY)) {
+ listOfCarbonDimPartOfKeyGen.add(carbonDim);
+ }
+
+ }
+ return listOfCarbonDimPartOfKeyGen;
+ }
+
+ private static void getEndKeyWithFilter(
+ Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter, long[] endKey,
+ List<long[]> endKeyList) {
+ for (Map.Entry<CarbonDimension, List<DimColumnFilterInfo>> entry : dimensionFilter.entrySet()) {
+ List<DimColumnFilterInfo> values = entry.getValue();
+ if (null == values || !entry.getKey().hasEncoding(Encoding.DICTIONARY)) {
+ continue;
+ }
+ boolean isExcludeFilterPresent = false;
+ for (DimColumnFilterInfo info : values) {
+ if (!info.isIncludeFilter()) {
+ isExcludeFilterPresent = true;
+ }
+ }
+ if (isExcludeFilterPresent) {
+ continue;
+ }
+
+ for (DimColumnFilterInfo info : values) {
+ if (endKey[entry.getKey().getKeyOrdinal()] > info.getFilterList()
+ .get(info.getFilterList().size() - 1)) {
+ endKey[entry.getKey().getKeyOrdinal()] =
+ info.getFilterList().get(info.getFilterList().size() - 1);
+ }
+ }
+ long[] newEndKey = new long[endKey.length];
+ System.arraycopy(endKey, 0, newEndKey, 0, endKey.length);
+ endKeyList.add(newEndKey);
+ }
+
+ }
+
+ /**
+ * This API will get the max value of surrogate key which will be used for
+ * determining the end key of particular btree.
+ *
+ * @param dimCarinality
+ */
+ private static long getMaxValue(CarbonDimension carbonDimension, int[] dimCarinality) {
+ // Get data from all the available slices of the table
+ if (null != dimCarinality) {
+ return dimCarinality[carbonDimension.getKeyOrdinal()];
+ }
+ return -1;
+ }
+
+ /**
+ * @param tableIdentifier
+ * @param carbonDimension
+ * @return
+ */
+ public static Dictionary getForwardDictionaryCache(AbsoluteTableIdentifier tableIdentifier,
+ CarbonDimension carbonDimension) throws IOException {
+ DictionaryColumnUniqueIdentifier dictionaryColumnUniqueIdentifier =
+ new DictionaryColumnUniqueIdentifier(tableIdentifier.getCarbonTableIdentifier(),
+ carbonDimension.getColumnIdentifier(), carbonDimension.getDataType());
+ CacheProvider cacheProvider = CacheProvider.getInstance();
+ Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache =
+ cacheProvider.createCache(CacheType.FORWARD_DICTIONARY, tableIdentifier.getStorePath());
+ // get the forward dictionary object
+ return forwardDictionaryCache.get(dictionaryColumnUniqueIdentifier);
+ }
+
+ public static IndexKey createIndexKeyFromResolvedFilterVal(long[] startOrEndKey,
+ KeyGenerator keyGenerator, byte[] startOrEndKeyForNoDictDimension) {
+ IndexKey indexKey = null;
+ try {
+ indexKey =
+ new IndexKey(keyGenerator.generateKey(startOrEndKey), startOrEndKeyForNoDictDimension);
+ } catch (KeyGenException e) {
+ LOGGER.error(e.getMessage());
+ }
+ return indexKey;
+ }
+
+ /**
+ * API will create an filter executer tree based on the filter resolver
+ *
+ * @param filterExpressionResolverTree
+ * @param segmentProperties
+ * @return
+ */
+ public static FilterExecuter getFilterExecuterTree(
+ FilterResolverIntf filterExpressionResolverTree, SegmentProperties segmentProperties,
+ Map<Integer, GenericQueryType> complexDimensionInfoMap) {
+ return createFilterExecuterTree(filterExpressionResolverTree, segmentProperties,
+ complexDimensionInfoMap);
+ }
+
+ /**
+ * API will prepare the Keys from the surrogates of particular filter resolver
+ *
+ * @param filterValues
+ * @param segmentProperties
+ * @param dimension
+ * @param dimColumnExecuterInfo
+ */
+ public static void prepareKeysFromSurrogates(DimColumnFilterInfo filterValues,
+ SegmentProperties segmentProperties, CarbonDimension dimension,
+ DimColumnExecuterFilterInfo dimColumnExecuterInfo) {
+ byte[][] keysBasedOnFilter = getKeyArray(filterValues, dimension, segmentProperties);
+ dimColumnExecuterInfo.setFilterKeys(keysBasedOnFilter);
+
+ }
+
+ /**
+ * method will create a default end key in case of no end key is been derived using existing
+ * filter or in case of non filter queries.
+ *
+ * @param segmentProperties
+ * @return
+ * @throws KeyGenException
+ */
+ public static IndexKey prepareDefaultEndIndexKey(SegmentProperties segmentProperties)
+ throws KeyGenException {
+ long[] dictionarySurrogateKey =
+ new long[segmentProperties.getDimensions().size() - segmentProperties
+ .getNumberOfNoDictionaryDimension()];
+ int index = 0;
+ int[] dimColumnsCardinality = segmentProperties.getDimColumnsCardinality();
+ for (int i = 0; i < dimColumnsCardinality.length; i++) {
+ dictionarySurrogateKey[index++] = dimColumnsCardinality[i];
+ }
+ IndexKey endIndexKey;
+ byte[] dictionaryendMdkey =
+ segmentProperties.getDimensionKeyGenerator().generateKey(dictionarySurrogateKey);
+ byte[] noDictionaryEndKeyBuffer = getNoDictionaryDefaultEndKey(segmentProperties);
+ endIndexKey = new IndexKey(dictionaryendMdkey, noDictionaryEndKeyBuffer);
+ return endIndexKey;
+ }
+
+ public static byte[] getNoDictionaryDefaultEndKey(SegmentProperties segmentProperties) {
+ // in case of non filter query when no dictionary columns are present we
+ // need to set the default end key, as for non filter query
+ // we need to get the last
+ // block of the btree so we are setting the max byte value in the end key
+ ByteBuffer noDictionaryEndKeyBuffer = ByteBuffer.allocate(
+ (segmentProperties.getNumberOfNoDictionaryDimension()
+ * CarbonCommonConstants.SHORT_SIZE_IN_BYTE) + segmentProperties
+ .getNumberOfNoDictionaryDimension());
+ // end key structure will be
+ //<Offset of first No Dictionary key in 2 Bytes><Offset of second No Dictionary key in 2 Bytes>
+ //<Offset of n No Dictionary key in 2 Bytes><first no dictionary column value>
+ // <second no dictionary column value> <N no dictionary column value>
+ //example if we have 2 no dictionary column
+ //<[0,4,0,5,127,127]>
+ short startPoint = (short) (segmentProperties.getNumberOfNoDictionaryDimension()
+ * CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+ for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
+ noDictionaryEndKeyBuffer.putShort((startPoint));
+ startPoint++;
+ }
+ for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
+ noDictionaryEndKeyBuffer.put((byte) 127);
+ }
+ return noDictionaryEndKeyBuffer.array();
+ }
+
+ /**
+ * method will create a default end key in case of no end key is been
+ * derived using existing filter or in case of non filter queries.
+ *
+ * @param segmentProperties
+ * @return
+ * @throws KeyGenException
+ */
+ public static IndexKey prepareDefaultStartIndexKey(SegmentProperties segmentProperties)
+ throws KeyGenException {
+ IndexKey startIndexKey;
+ long[] dictionarySurrogateKey =
+ new long[segmentProperties.getDimensions().size() - segmentProperties
+ .getNumberOfNoDictionaryDimension()];
+ byte[] dictionaryStartMdkey =
+ segmentProperties.getDimensionKeyGenerator().generateKey(dictionarySurrogateKey);
+ byte[] noDictionaryStartKeyArray = getNoDictionaryDefaultStartKey(segmentProperties);
+
+ startIndexKey = new IndexKey(dictionaryStartMdkey, noDictionaryStartKeyArray);
+ return startIndexKey;
+ }
+
+ public static byte[] getNoDictionaryDefaultStartKey(SegmentProperties segmentProperties) {
+ // in case of non filter query when no dictionary columns are present we
+ // need to set the default start key, as for non filter query we need to get the first
+ // block of the btree so we are setting the least byte value in the start key
+ ByteBuffer noDictionaryStartKeyBuffer = ByteBuffer.allocate(
+ (segmentProperties.getNumberOfNoDictionaryDimension()
+ * CarbonCommonConstants.SHORT_SIZE_IN_BYTE) + segmentProperties
+ .getNumberOfNoDictionaryDimension());
+ // end key structure will be
+ //<Offset of first No Dictionary key in 2 Bytes><Offset of second No Dictionary key in 2 Bytes>
+ //<Offset of n No Dictionary key in 2 Bytes><first no dictionary column value>
+ // <second no dictionary column value> <N no dictionary column value>
+ //example if we have 2 no dictionary column
+ //<[0,4,0,5,0,0]>
+ short startPoint = (short) (segmentProperties.getNumberOfNoDictionaryDimension()
+ * CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+ for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
+ noDictionaryStartKeyBuffer.putShort((startPoint));
+ startPoint++;
+ }
+ for (int i = 0; i < segmentProperties.getNumberOfNoDictionaryDimension(); i++) {
+ noDictionaryStartKeyBuffer.put((byte) 0);
+ }
+ return noDictionaryStartKeyBuffer.array();
+ }
+
+ public static int compareFilterKeyBasedOnDataType(String dictionaryVal, String memberVal,
+ DataType dataType) {
+ try {
+ switch (dataType) {
+ case SHORT:
+ return Short.compare((Short.parseShort(dictionaryVal)), (Short.parseShort(memberVal)));
+ case INT:
+ return Integer.compare((Integer.parseInt(dictionaryVal)), (Integer.parseInt(memberVal)));
+ case DOUBLE:
+ return Double
+ .compare((Double.parseDouble(dictionaryVal)), (Double.parseDouble(memberVal)));
+ case LONG:
+ return Long.compare((Long.parseLong(dictionaryVal)), (Long.parseLong(memberVal)));
+ case BOOLEAN:
+ return Boolean
+ .compare((Boolean.parseBoolean(dictionaryVal)), (Boolean.parseBoolean(memberVal)));
+ case DATE:
+ case TIMESTAMP:
+ SimpleDateFormat parser = new SimpleDateFormat(CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
+ Date dateToStr;
+ Date dictionaryDate;
+ dateToStr = parser.parse(memberVal);
+ dictionaryDate = parser.parse(dictionaryVal);
+ return dictionaryDate.compareTo(dateToStr);
+
+ case DECIMAL:
+ java.math.BigDecimal javaDecValForDictVal = new java.math.BigDecimal(dictionaryVal);
+ java.math.BigDecimal javaDecValForMemberVal = new java.math.BigDecimal(memberVal);
+ return javaDecValForDictVal.compareTo(javaDecValForMemberVal);
+ default:
+ return -1;
+ }
+ } catch (Exception e) {
+ return -1;
+ }
+ }
+
+ /**
+ * method will set the start and end key for as per the filter resolver tree
+ * utilized visitor pattern inorder to populate the start and end key population.
+ *
+ * @param segmentProperties
+ * @param filterResolver
+ * @param listOfStartEndKeys
+ */
+ public static void traverseResolverTreeAndGetStartAndEndKey(SegmentProperties segmentProperties,
+ FilterResolverIntf filterResolver, List<IndexKey> listOfStartEndKeys) {
+ IndexKey searchStartKey = null;
+ IndexKey searchEndKey = null;
+ long[] startKey = new long[segmentProperties.getDimensionKeyGenerator().getDimCount()];
+ long[] endKey = new long[segmentProperties.getDimensionKeyGenerator().getDimCount()];
+ List<byte[]> listOfStartKeyByteArray =
+ new ArrayList<byte[]>(segmentProperties.getNumberOfNoDictionaryDimension());
+ List<byte[]> listOfEndKeyByteArray =
+ new ArrayList<byte[]>(segmentProperties.getNumberOfNoDictionaryDimension());
+ SortedMap<Integer, byte[]> setOfStartKeyByteArray = new TreeMap<Integer, byte[]>();
+ SortedMap<Integer, byte[]> setOfEndKeyByteArray = new TreeMap<Integer, byte[]>();
+ SortedMap<Integer, byte[]> defaultStartValues = new TreeMap<Integer, byte[]>();
+ SortedMap<Integer, byte[]> defaultEndValues = new TreeMap<Integer, byte[]>();
+ List<long[]> startKeyList = new ArrayList<long[]>();
+ List<long[]> endKeyList = new ArrayList<long[]>();
+ traverseResolverTreeAndPopulateStartAndEndKeys(filterResolver, segmentProperties, startKey,
+ setOfStartKeyByteArray, endKey, setOfEndKeyByteArray,
+ startKeyList, endKeyList);
+ if (endKeyList.size() > 0) {
+ //get the new end key from list
+ for (int i = 0; i < endKey.length; i++) {
+ long[] endkeyColumnLevel = new long[endKeyList.size()];
+ int j = 0;
+ for (long[] oneEndKey : endKeyList) {
+ //get each column level end key
+ endkeyColumnLevel[j++] = oneEndKey[i];
+ }
+ Arrays.sort(endkeyColumnLevel);
+ // get the max one as end of this column level
+ endKey[i] = endkeyColumnLevel[endkeyColumnLevel.length - 1];
+ }
+ }
+
+ if (startKeyList.size() > 0) {
+ //get the new start key from list
+ for (int i = 0; i < startKey.length; i++) {
+ long[] startkeyColumnLevel = new long[startKeyList.size()];
+ int j = 0;
+ for (long[] oneStartKey : startKeyList) {
+ //get each column level start key
+ startkeyColumnLevel[j++] = oneStartKey[i];
+ }
+ Arrays.sort(startkeyColumnLevel);
+ // get the min - 1 as start of this column level, for example if a block contains 5,6
+ // the filter is 6, but that block's start key is 5, if not -1, this block will missing.
+ startKey[i] = startkeyColumnLevel[0] - 1;
+ }
+ }
+
+ fillDefaultStartValue(defaultStartValues, segmentProperties);
+ fillDefaultEndValue(defaultEndValues, segmentProperties);
+ fillNullValuesStartIndexWithDefaultKeys(setOfStartKeyByteArray, segmentProperties);
+ fillNullValuesEndIndexWithDefaultKeys(setOfEndKeyByteArray, segmentProperties);
+ pruneStartAndEndKeys(setOfStartKeyByteArray, listOfStartKeyByteArray);
+ pruneStartAndEndKeys(setOfEndKeyByteArray, listOfEndKeyByteArray);
+
+ searchStartKey = FilterUtil
+ .createIndexKeyFromResolvedFilterVal(startKey, segmentProperties.getDimensionKeyGenerator(),
+ FilterUtil.getKeyWithIndexesAndValues(listOfStartKeyByteArray));
+
+ searchEndKey = FilterUtil
+ .createIndexKeyFromResolvedFilterVal(endKey, segmentProperties.getDimensionKeyGenerator(),
+ FilterUtil.getKeyWithIndexesAndValues(listOfEndKeyByteArray));
+ listOfStartEndKeys.add(searchStartKey);
+ listOfStartEndKeys.add(searchEndKey);
+
+ }
+
+ private static int compareFilterMembersBasedOnActualDataType(String filterMember1,
+ String filterMember2, DataType dataType) {
+ try {
+ switch (dataType) {
+ case SHORT:
+ case INT:
+ case LONG:
+ case DOUBLE:
+
+ if (CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(filterMember1)) {
+ return 1;
+ }
+ Double d1 = Double.parseDouble(filterMember1);
+ Double d2 = Double.parseDouble(filterMember2);
+ return d1.compareTo(d2);
+ case DECIMAL:
+ if (CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(filterMember1)) {
+ return 1;
+ }
+ java.math.BigDecimal val1 = new BigDecimal(filterMember1);
+ java.math.BigDecimal val2 = new BigDecimal(filterMember2);
+ return val1.compareTo(val2);
+ case DATE:
+ case TIMESTAMP:
+ if (CarbonCommonConstants.MEMBER_DEFAULT_VAL.equals(filterMember1)) {
+ return 1;
+ }
+ SimpleDateFormat parser = new SimpleDateFormat(CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
+ Date date1 = null;
+ Date date2 = null;
+ date1 = parser.parse(filterMember1);
+ date2 = parser.parse(filterMember2);
+ return date1.compareTo(date2);
+ case STRING:
+ default:
+ return filterMember1.compareTo(filterMember2);
+ }
+ } catch (Exception e) {
+ return -1;
+ }
+ }
+
+ private static void fillNullValuesStartIndexWithDefaultKeys(
+ SortedMap<Integer, byte[]> setOfStartKeyByteArray, SegmentProperties segmentProperties) {
+ List<CarbonDimension> allDimension = segmentProperties.getDimensions();
+ for (CarbonDimension dimension : allDimension) {
+ if (CarbonUtil.hasEncoding(dimension.getEncoder(), Encoding.DICTIONARY)) {
+ continue;
+ }
+ if (null == setOfStartKeyByteArray.get(dimension.getOrdinal())) {
+ setOfStartKeyByteArray.put(dimension.getOrdinal(), new byte[] { 0 });
+ }
+
+ }
+ }
+
+ private static void fillNullValuesEndIndexWithDefaultKeys(
+ SortedMap<Integer, byte[]> setOfStartKeyByteArray, SegmentProperties segmentProperties) {
+ List<CarbonDimension> allDimension = segmentProperties.getDimensions();
+ for (CarbonDimension dimension : allDimension) {
+ if (CarbonUtil.hasEncoding(dimension.getEncoder(), Encoding.DICTIONARY)) {
+ continue;
+ }
+ if (null == setOfStartKeyByteArray.get(dimension.getOrdinal())) {
+ setOfStartKeyByteArray.put(dimension.getOrdinal(), new byte[] { 127 });
+ }
+
+ }
+ }
+
+ private static void pruneStartAndEndKeys(SortedMap<Integer, byte[]> setOfStartKeyByteArray,
+ List<byte[]> listOfStartKeyByteArray) {
+ for (Map.Entry<Integer, byte[]> entry : setOfStartKeyByteArray.entrySet()) {
+ listOfStartKeyByteArray.add(entry.getValue());
+ }
+ }
+
+ private static void fillDefaultStartValue(SortedMap<Integer, byte[]> setOfStartKeyByteArray,
+ SegmentProperties segmentProperties) {
+ List<CarbonDimension> allDimension = segmentProperties.getDimensions();
+ for (CarbonDimension dimension : allDimension) {
+ if (CarbonUtil.hasEncoding(dimension.getEncoder(), Encoding.DICTIONARY)) {
+ continue;
+ }
+ setOfStartKeyByteArray.put(dimension.getOrdinal(), new byte[] { 0 });
+ }
+
+ }
+
+ private static void fillDefaultEndValue(SortedMap<Integer, byte[]> setOfEndKeyByteArray,
+ SegmentProperties segmentProperties) {
+ List<CarbonDimension> allDimension = segmentProperties.getDimensions();
+ for (CarbonDimension dimension : allDimension) {
+ if (CarbonUtil.hasEncoding(dimension.getEncoder(), Encoding.DICTIONARY)) {
+ continue;
+ }
+ setOfEndKeyByteArray.put(dimension.getOrdinal(), new byte[] { 127 });
+ }
+ }
+
+ private static void traverseResolverTreeAndPopulateStartAndEndKeys(
+ FilterResolverIntf filterResolverTree, SegmentProperties segmentProperties, long[] startKeys,
+ SortedMap<Integer, byte[]> setOfStartKeyByteArray, long[] endKeys,
+ SortedMap<Integer, byte[]> setOfEndKeyByteArray, List<long[]> startKeyList,
+ List<long[]> endKeyList) {
+ if (null == filterResolverTree) {
+ return;
+ }
+ traverseResolverTreeAndPopulateStartAndEndKeys(filterResolverTree.getLeft(),
+ segmentProperties, startKeys, setOfStartKeyByteArray, endKeys, setOfEndKeyByteArray,
+ startKeyList, endKeyList);
+ filterResolverTree.getStartKey(startKeys, setOfStartKeyByteArray, startKeyList);
+ filterResolverTree.getEndKey(segmentProperties, endKeys, setOfEndKeyByteArray,
+ endKeyList);
+
+ traverseResolverTreeAndPopulateStartAndEndKeys(filterResolverTree.getRight(),
+ segmentProperties, startKeys, setOfStartKeyByteArray, endKeys, setOfEndKeyByteArray,
+ startKeyList, endKeyList);
+ }
+
+ /**
+ * Method will find whether the expression needs to be resolved, this can happen
+ * if the expression is exclude and data type is null(mainly in IS NOT NULL filter scenario)
+ *
+ * @param rightExp
+ * @param isIncludeFilter
+ * @return
+ */
+ public static boolean isExpressionNeedsToResolved(Expression rightExp, boolean isIncludeFilter) {
+ if (!isIncludeFilter && rightExp instanceof LiteralExpression && (
+ DataType.NULL == ((LiteralExpression) rightExp)
+ .getLiteralExpDataType())) {
+ return true;
+ }
+ for (Expression child : rightExp.getChildren()) {
+ if (isExpressionNeedsToResolved(child, isIncludeFilter)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * This method will print the error log.
+ *
+ * @param e
+ */
+ public static void logError(Throwable e, boolean invalidRowsPresent) {
+ if (!invalidRowsPresent) {
+ LOGGER.error(e, CarbonCommonConstants.FILTER_INVALID_MEMBER + e.getMessage());
+ }
+ }
+
+ /**
+ * This method will compare double values for its equality and also it will preserve
+ * the -0.0 and 0.0 equality as per == ,also preserve NaN equality check as per
+ * java.lang.Double.equals()
+ *
+ * @param d1 double value for equality check
+ * @param d2 double value for equality check
+ * @return boolean after comparing two double values.
+ */
+ public static boolean nanSafeEqualsDoubles(Double d1, Double d2) {
+ if ((d1.doubleValue() == d2.doubleValue()) || (Double.isNaN(d1) && Double.isNaN(d2))) {
+ return true;
+ }
+ return false;
+
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/GenericQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/GenericQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/GenericQueryType.java
new file mode 100644
index 0000000..49c9155
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/GenericQueryType.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.filter;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+
+public interface GenericQueryType {
+
+ String getName();
+
+ void setName(String name);
+
+ String getParentname();
+
+ void setParentname(String parentname);
+
+ void addChildren(GenericQueryType children);
+
+ int getColsCount();
+
+ void parseBlocksAndReturnComplexColumnByteArray(DimensionColumnDataChunk[] dimensionDataChunks,
+ int rowNumber, DataOutputStream dataOutputStream) throws IOException;
+
+ void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder) throws IOException;
+
+ Object getDataBasedOnDataTypeFromSurrogates(ByteBuffer surrogateData);
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java
new file mode 100644
index 0000000..1ab84bc
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/AndFilterExecuterImpl.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter.executer;
+
+import java.io.IOException;
+import java.util.BitSet;
+
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+
+public class AndFilterExecuterImpl implements FilterExecuter {
+
+ private FilterExecuter leftExecuter;
+ private FilterExecuter rightExecuter;
+
+ public AndFilterExecuterImpl(FilterExecuter leftExecuter, FilterExecuter rightExecuter) {
+ this.leftExecuter = leftExecuter;
+ this.rightExecuter = rightExecuter;
+ }
+
+ @Override public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
+ throws FilterUnsupportedException, IOException {
+ BitSet leftFilters = leftExecuter.applyFilter(blockChunkHolder);
+ if (leftFilters.isEmpty()) {
+ return leftFilters;
+ }
+ BitSet rightFilter = rightExecuter.applyFilter(blockChunkHolder);
+ if (rightFilter.isEmpty()) {
+ return rightFilter;
+ }
+ leftFilters.and(rightFilter);
+ return leftFilters;
+ }
+
+ @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+ BitSet leftFilters = leftExecuter.isScanRequired(blockMaxValue, blockMinValue);
+ if (leftFilters.isEmpty()) {
+ return leftFilters;
+ }
+ BitSet rightFilter = rightExecuter.isScanRequired(blockMaxValue, blockMinValue);
+ if (rightFilter.isEmpty()) {
+ return rightFilter;
+ }
+ leftFilters.and(rightFilter);
+ return leftFilters;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/DimColumnExecuterFilterInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/DimColumnExecuterFilterInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/DimColumnExecuterFilterInfo.java
new file mode 100644
index 0000000..0bb7ce3
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/DimColumnExecuterFilterInfo.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter.executer;
+
+public class DimColumnExecuterFilterInfo {
+
+ byte[][] filterKeys;
+
+ public void setFilterKeys(byte[][] filterKeys) {
+ this.filterKeys = filterKeys;
+ }
+
+ public byte[][] getFilterKeys() {
+ return filterKeys;
+ }
+}
[41/57] [abbrv] incubator-carbondata git commit: move load
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
index 81628b3..788488a 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
@@ -18,10 +18,10 @@
*/
package org.apache.carbondata.core.scan.processor;
+import org.apache.carbondata.core.datastorage.FileHolder;
import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
-import org.apache.carbondata.core.datastorage.FileHolder;
/**
* Block chunk holder which will hold the dimension and
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java
index 4de9685..6df1b98 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java
@@ -21,11 +21,11 @@ package org.apache.carbondata.core.scan.processor.impl;
import java.util.ArrayList;
import java.util.List;
-import org.apache.carbondata.core.stats.QueryStatisticsModel;
import org.apache.carbondata.core.datastorage.FileHolder;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.processor.AbstractDataBlockIterator;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
+import org.apache.carbondata.core.stats.QueryStatisticsModel;
/**
* Below class will be used to process the block for detail query
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
index 82649b4..cbe9e5f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
@@ -25,18 +25,18 @@ import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.util.Map;
-import org.apache.carbondata.core.update.data.BlockletLevelDeleteDeltaDataCache;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
import org.apache.carbondata.core.path.CarbonTablePath;
-import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.filter.GenericQueryType;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
+import org.apache.carbondata.core.update.data.BlockletLevelDeleteDeltaDataCache;
+import org.apache.carbondata.core.util.CarbonUtil;
/**
* Scanned result class which will store and provide the result on request
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
index 88c0144..69bd273 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
@@ -24,22 +24,22 @@ import java.util.concurrent.ExecutorService;
import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.DataRefNode;
-import org.apache.carbondata.core.datastore.DataRefNodeFinder;
-import org.apache.carbondata.core.datastore.impl.btree.BTreeDataRefNodeFinder;
-import org.apache.carbondata.core.stats.QueryStatistic;
-import org.apache.carbondata.core.stats.QueryStatisticsConstants;
-import org.apache.carbondata.core.stats.QueryStatisticsModel;
-import org.apache.carbondata.core.stats.QueryStatisticsRecorder;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.FileHolder;
import org.apache.carbondata.core.datastorage.impl.FileFactory;
-import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.datastore.DataRefNodeFinder;
+import org.apache.carbondata.core.datastore.impl.btree.BTreeDataRefNodeFinder;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.core.scan.processor.AbstractDataBlockIterator;
import org.apache.carbondata.core.scan.processor.impl.DataBlockIteratorImpl;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
+import org.apache.carbondata.core.stats.QueryStatistic;
+import org.apache.carbondata.core.stats.QueryStatisticsConstants;
+import org.apache.carbondata.core.stats.QueryStatisticsModel;
+import org.apache.carbondata.core.stats.QueryStatisticsRecorder;
+import org.apache.carbondata.core.util.CarbonProperties;
/**
* In case of detail query we cannot keep all the records in memory so for
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
index 32b3ac2..2f62523 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/AbstractBlockletScanner.java
@@ -20,16 +20,16 @@ package org.apache.carbondata.core.scan.scanner;
import java.io.IOException;
-import org.apache.carbondata.core.update.data.BlockletDeleteDeltaCacheLoader;
-import org.apache.carbondata.core.update.data.DeleteDeltaCacheLoaderIntf;
-import org.apache.carbondata.core.stats.QueryStatistic;
-import org.apache.carbondata.core.stats.QueryStatisticsConstants;
-import org.apache.carbondata.core.stats.QueryStatisticsModel;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.stats.QueryStatistic;
+import org.apache.carbondata.core.stats.QueryStatisticsConstants;
+import org.apache.carbondata.core.stats.QueryStatisticsModel;
+import org.apache.carbondata.core.update.data.BlockletDeleteDeltaCacheLoader;
+import org.apache.carbondata.core.update.data.DeleteDeltaCacheLoaderIntf;
/**
* Blocklet scanner class to process the block
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
index 48c9af2..3d17985 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
@@ -22,17 +22,10 @@ package org.apache.carbondata.core.scan.scanner.impl;
import java.io.IOException;
import java.util.BitSet;
-import org.apache.carbondata.core.update.data.BlockletDeleteDeltaCacheLoader;
-import org.apache.carbondata.core.update.data.DeleteDeltaCacheLoaderIntf;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
-import org.apache.carbondata.core.stats.QueryStatistic;
-import org.apache.carbondata.core.stats.QueryStatisticsConstants;
-import org.apache.carbondata.core.stats.QueryStatisticsModel;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.FileHolder;
-import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
@@ -40,6 +33,13 @@ import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
import org.apache.carbondata.core.scan.result.AbstractScannedResult;
import org.apache.carbondata.core.scan.result.impl.FilterQueryScannedResult;
import org.apache.carbondata.core.scan.scanner.AbstractBlockletScanner;
+import org.apache.carbondata.core.stats.QueryStatistic;
+import org.apache.carbondata.core.stats.QueryStatisticsConstants;
+import org.apache.carbondata.core.stats.QueryStatisticsModel;
+import org.apache.carbondata.core.update.data.BlockletDeleteDeltaCacheLoader;
+import org.apache.carbondata.core.update.data.DeleteDeltaCacheLoaderIntf;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.CarbonUtil;
/**
* Below class will be used for filter query processing
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/NonFilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/NonFilterScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/NonFilterScanner.java
index f07c2be..cc0838f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/NonFilterScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/NonFilterScanner.java
@@ -18,10 +18,10 @@
*/
package org.apache.carbondata.core.scan.scanner.impl;
-import org.apache.carbondata.core.stats.QueryStatisticsModel;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.result.impl.NonFilterQueryScannedResult;
import org.apache.carbondata.core.scan.scanner.AbstractBlockletScanner;
+import org.apache.carbondata.core.stats.QueryStatisticsModel;
/**
* Non filter processor which will be used for non filter query
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/service/CarbonCommonFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/service/CarbonCommonFactory.java b/core/src/main/java/org/apache/carbondata/core/service/CarbonCommonFactory.java
index 3c33867..bc63d01 100644
--- a/core/src/main/java/org/apache/carbondata/core/service/CarbonCommonFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/service/CarbonCommonFactory.java
@@ -21,9 +21,6 @@ package org.apache.carbondata.core.service;
import org.apache.carbondata.core.service.impl.ColumnUniqueIdGenerator;
import org.apache.carbondata.core.service.impl.DictionaryFactory;
import org.apache.carbondata.core.service.impl.PathFactory;
-import org.apache.carbondata.core.service.ColumnUniqueIdService;
-import org.apache.carbondata.core.service.DictionaryService;
-import org.apache.carbondata.core.service.PathService;
/**
* Interface to get services
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImpl.java b/core/src/main/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImpl.java
index 35812dd..d41c57d 100644
--- a/core/src/main/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImpl.java
@@ -27,10 +27,10 @@ import java.util.concurrent.ConcurrentHashMap;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import static org.apache.carbondata.core.util.CarbonUtil.printLine;
-
import org.apache.commons.lang3.StringUtils;
+import static org.apache.carbondata.core.util.CarbonUtil.printLine;
+
/**
* Class will be used to record and log the query statistics
*/
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java
index 59e8c25..3af7e65 100644
--- a/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java
@@ -20,26 +20,32 @@ package org.apache.carbondata.core.update;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.path.CarbonStorePath;
-import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastorage.impl.FileFactory;
-import org.apache.carbondata.core.load.LoadMetadataDetails;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.path.CarbonStorePath;
+import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.update.data.BlockMappingVO;
import org.apache.carbondata.core.update.data.RowCountDetailsVO;
+import org.apache.carbondata.core.updatestatus.LoadMetadataDetails;
import org.apache.carbondata.core.updatestatus.SegmentStatusManager;
import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager;
+import org.apache.carbondata.core.updatestatus.locks.ICarbonLock;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.updatestatus.locks.ICarbonLock;
/**
@@ -407,75 +413,6 @@ public class CarbonUpdateUtil {
}
- public static String getLatestBlockNameForSegment(String segmentId, CarbonTablePath tablePath) {
- String segmentDirPath = tablePath.getCarbonDataDirectoryPath("0", segmentId);
-
- // scan all the carbondata files and get the latest task ID.
- CarbonFile segment =
- FileFactory.getCarbonFile(segmentDirPath, FileFactory.getFileType(segmentDirPath));
-
- CarbonFile[] dataFiles = segment.listFiles(new CarbonFileFilter() {
- @Override public boolean accept(CarbonFile file) {
- int max = 0;
- if (file.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT)) {
- int taskNumber = Integer.parseInt(CarbonTablePath.DataFileUtil.getTaskNo(file.getName()));
- if (taskNumber >= max) {
- return true;
- }
- }
- return false;
- }
- });
-
- // get the latest among the data files. highest task number will be at the last.
- return dataFiles[dataFiles.length - 1].getName();
- }
-
- /**
- * This method will convert a given timestamp to long value and then to string back
- *
- * @param factTimeStamp
- * @return
- */
- public static String convertTimeStampToString(String factTimeStamp) {
- SimpleDateFormat parser = new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP);
- Date dateToStr = null;
- try {
- dateToStr = parser.parse(factTimeStamp);
- return Long.toString(dateToStr.getTime());
- } catch (ParseException e) {
- LOGGER.error("Cannot convert" + factTimeStamp + " to Time/Long type value" + e.getMessage());
- return null;
- }
- }
-
- /**
- * This method will convert a given timestamp to long value and then to string back
- *
- * @param factTimeStamp
- * @return
- */
- public static long convertTimeStampToLong(String factTimeStamp) {
- SimpleDateFormat parser = new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP_MILLIS);
- Date dateToStr = null;
- try {
- dateToStr = parser.parse(factTimeStamp);
- return dateToStr.getTime();
- } catch (ParseException e) {
- LOGGER.error("Cannot convert" + factTimeStamp + " to Time/Long type value" + e.getMessage());
- parser = new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP);
- try {
- dateToStr = parser.parse(factTimeStamp);
- return dateToStr.getTime();
- } catch (ParseException e1) {
- LOGGER
- .error("Cannot convert" + factTimeStamp + " to Time/Long type value" + e1.getMessage());
- return 0;
- }
- }
- }
-
-
/**
* Handling of the clean up of old carbondata files, index files , delte delta,
* update status files.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/update/data/BlockletDeleteDeltaCacheLoader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/update/data/BlockletDeleteDeltaCacheLoader.java b/core/src/main/java/org/apache/carbondata/core/update/data/BlockletDeleteDeltaCacheLoader.java
index f7c2a2d..30aeab8 100644
--- a/core/src/main/java/org/apache/carbondata/core/update/data/BlockletDeleteDeltaCacheLoader.java
+++ b/core/src/main/java/org/apache/carbondata/core/update/data/BlockletDeleteDeltaCacheLoader.java
@@ -19,8 +19,8 @@ package org.apache.carbondata.core.update.data;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/updatestatus/LoadMetadataDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/LoadMetadataDetails.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/LoadMetadataDetails.java
new file mode 100644
index 0000000..89b0604
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/LoadMetadataDetails.java
@@ -0,0 +1,344 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.updatestatus;
+
+import java.io.Serializable;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+
+public class LoadMetadataDetails implements Serializable {
+
+ private static final long serialVersionUID = 1106104914918491724L;
+ private String timestamp;
+ private String loadStatus;
+ private String loadName;
+ private String partitionCount;
+ private String isDeleted = CarbonCommonConstants.KEYWORD_FALSE;
+
+ // update delta end timestamp
+ private String updateDeltaEndTimestamp = "";
+
+ // update delta start timestamp
+ private String updateDeltaStartTimestamp = "";
+
+ // this will represent the update status file name at that point of time.
+ private String updateStatusFileName = "";
+
+ /**
+ * LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(LoadMetadataDetails.class.getName());
+
+ // dont remove static as the write will fail.
+ private static final SimpleDateFormat parser =
+ new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP);
+ /**
+ * Segment modification or deletion time stamp
+ */
+ private String modificationOrdeletionTimesStamp;
+ private String loadStartTime;
+
+ private String mergedLoadName;
+ /**
+ * visibility is used to determine whether to the load is visible or not.
+ */
+ private String visibility = "true";
+
+ /**
+ * To know if the segment is a major compacted segment or not.
+ */
+ private String majorCompacted;
+
+ public String getPartitionCount() {
+ return partitionCount;
+ }
+
+ public void setPartitionCount(String partitionCount) {
+ this.partitionCount = partitionCount;
+ }
+
+ public long getLoadEndTime() {
+ return convertTimeStampToLong(timestamp);
+ }
+
+ public void setLoadEndTime(long timestamp) {
+ this.timestamp = getTimeStampConvertion(timestamp);;
+ }
+
+ public String getLoadStatus() {
+ return loadStatus;
+ }
+
+ public void setLoadStatus(String loadStatus) {
+ this.loadStatus = loadStatus;
+ }
+
+ public String getLoadName() {
+ return loadName;
+ }
+
+ public void setLoadName(String loadName) {
+ this.loadName = loadName;
+ }
+
+ /**
+ * @return the modificationOrdeletionTimesStamp
+ */
+ public long getModificationOrdeletionTimesStamp() {
+ if(null == modificationOrdeletionTimesStamp) {
+ return 0;
+ }
+ return convertTimeStampToLong(modificationOrdeletionTimesStamp);
+ }
+
+ /**
+ * @param modificationOrdeletionTimesStamp the modificationOrdeletionTimesStamp to set
+ */
+ public void setModificationOrdeletionTimesStamp(long modificationOrdeletionTimesStamp) {
+ this.modificationOrdeletionTimesStamp =
+ getTimeStampConvertion(modificationOrdeletionTimesStamp);
+ }
+
+ /* (non-Javadoc)
+ * @see java.lang.Object#hashCode()
+ */
+ @Override public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((loadName == null) ? 0 : loadName.hashCode());
+ return result;
+ }
+
+ /* (non-Javadoc)
+ * @see java.lang.Object#equals(java.lang.Object)
+ */
+ @Override public boolean equals(Object obj) {
+ if (obj == null) {
+ return false;
+
+ }
+ if (!(obj instanceof LoadMetadataDetails)) {
+ return false;
+ }
+ LoadMetadataDetails other = (LoadMetadataDetails) obj;
+ if (loadName == null) {
+ if (other.loadName != null) {
+ return false;
+ }
+ } else if (!loadName.equals(other.loadName)) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * @return the startLoadTime
+ */
+ public long getLoadStartTime() {
+ return convertTimeStampToLong(loadStartTime);
+ }
+
+ /**
+ * return loadStartTime
+ *
+ * @return
+ */
+ public long getLoadStartTimeAsLong() {
+ return (!loadStartTime.isEmpty()) ? getTimeStamp(loadStartTime) : 0;
+ }
+
+ /**
+ * This method will convert a given timestamp to long value and then to string back
+ *
+ * @param factTimeStamp
+ * @return
+ */
+ private long convertTimeStampToLong(String factTimeStamp) {
+ SimpleDateFormat parser = new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP_MILLIS);
+ Date dateToStr = null;
+ try {
+ dateToStr = parser.parse(factTimeStamp);
+ return dateToStr.getTime();
+ } catch (ParseException e) {
+ LOGGER.error("Cannot convert" + factTimeStamp + " to Time/Long type value" + e.getMessage());
+ parser = new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP);
+ try {
+ dateToStr = parser.parse(factTimeStamp);
+ return dateToStr.getTime();
+ } catch (ParseException e1) {
+ LOGGER
+ .error("Cannot convert" + factTimeStamp + " to Time/Long type value" + e1.getMessage());
+ return 0;
+ }
+ }
+ }
+
+ /**
+ * returns load start time as long value
+ *
+ * @param loadStartTime
+ * @return
+ */
+ public Long getTimeStamp(String loadStartTime) {
+ Date dateToStr = null;
+ try {
+ dateToStr = parser.parse(loadStartTime);
+ return dateToStr.getTime() * 1000;
+ } catch (ParseException e) {
+ LOGGER.error("Cannot convert" + loadStartTime + " to Time/Long type value" + e.getMessage());
+ return null;
+ }
+ }
+
+ private String getTimeStampConvertion(long time) {
+ SimpleDateFormat sdf = new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP_MILLIS);
+ return sdf.format(time);
+ }
+
+ /**
+ * @param loadStartTime
+ */
+ public void setLoadStartTime(long loadStartTime) {
+ this.loadStartTime = getTimeStampConvertion(loadStartTime);
+ }
+
+ /**
+ * @return the mergedLoadName
+ */
+ public String getMergedLoadName() {
+ return mergedLoadName;
+ }
+
+ /**
+ * @param mergedLoadName the mergedLoadName to set
+ */
+ public void setMergedLoadName(String mergedLoadName) {
+ this.mergedLoadName = mergedLoadName;
+ }
+
+ /**
+ * @return the visibility
+ */
+ public String getVisibility() {
+ return visibility;
+ }
+
+ /**
+ * @param visibility the visibility to set
+ */
+ public void setVisibility(String visibility) {
+ this.visibility = visibility;
+ }
+
+ /**
+ * Return true if it is a major compacted segment.
+ * @return majorCompacted
+ */
+ public String isMajorCompacted() {
+ return majorCompacted;
+ }
+
+ /**
+ * Set true if it is a major compacted segment.
+ *
+ * @param majorCompacted
+ */
+ public void setMajorCompacted(String majorCompacted) {
+ this.majorCompacted = majorCompacted;
+ }
+
+ /**
+ * To get isDeleted property.
+ *
+ * @return isDeleted
+ */
+ public String getIsDeleted() {
+ return isDeleted;
+ }
+
+ /**
+ * To set isDeleted property.
+ *
+ * @param isDeleted
+ */
+ public void setIsDeleted(String isDeleted) {
+ this.isDeleted = isDeleted;
+ }
+
+ /**
+ * To get the update delta end timestamp
+ *
+ * @return updateDeltaEndTimestamp
+ */
+ public String getUpdateDeltaEndTimestamp() {
+ return updateDeltaEndTimestamp;
+ }
+
+ /**
+ * To set the update delta end timestamp
+ *
+ * @param updateDeltaEndTimestamp
+ */
+ public void setUpdateDeltaEndTimestamp(String updateDeltaEndTimestamp) {
+ this.updateDeltaEndTimestamp = updateDeltaEndTimestamp;
+ }
+
+ /**
+ * To get the update delta start timestamp
+ *
+ * @return updateDeltaStartTimestamp
+ */
+ public String getUpdateDeltaStartTimestamp() {
+ return updateDeltaStartTimestamp;
+ }
+
+ /**
+ * To set the update delta start timestamp
+ *
+ * @param updateDeltaStartTimestamp
+ */
+ public void setUpdateDeltaStartTimestamp(String updateDeltaStartTimestamp) {
+ this.updateDeltaStartTimestamp = updateDeltaStartTimestamp;
+ }
+
+ /**
+ * To get the updateStatusFileName
+ *
+ * @return updateStatusFileName
+ */
+ public String getUpdateStatusFileName() {
+ return updateStatusFileName;
+ }
+
+ /**
+ * To set the updateStatusFileName
+ *
+ * @param updateStatusFileName
+ */
+ public void setUpdateStatusFileName(String updateStatusFileName) {
+ this.updateStatusFileName = updateStatusFileName;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java
index e01ba6d..9ebbbfb 100644
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java
@@ -18,7 +18,14 @@
*/
package org.apache.carbondata.core.updatestatus;
-import java.io.*;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.Closeable;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
@@ -26,22 +33,21 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.atomic.AtomicFileOperations;
+import org.apache.carbondata.core.atomic.AtomicFileOperationsImpl;
+import org.apache.carbondata.core.atomic.FileWriteOperation;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
-import org.apache.carbondata.core.load.LoadMetadataDetails;
import org.apache.carbondata.core.update.CarbonUpdateUtil;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.atomic.AtomicFileOperations;
-import org.apache.carbondata.core.atomic.AtomicFileOperationsImpl;
-import org.apache.carbondata.core.atomic.FileWriteOperation;
import org.apache.carbondata.core.updatestatus.locks.CarbonLockFactory;
import org.apache.carbondata.core.updatestatus.locks.CarbonLockUtil;
import org.apache.carbondata.core.updatestatus.locks.ICarbonLock;
import org.apache.carbondata.core.updatestatus.locks.LockUsage;
+import org.apache.carbondata.core.util.CarbonUtil;
import com.google.gson.Gson;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java
index 4e44661..425b039 100644
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java
@@ -17,7 +17,14 @@
package org.apache.carbondata.core.updatestatus;
-import java.io.*;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.Closeable;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -25,26 +32,25 @@ import java.util.Map;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.path.CarbonStorePath;
-import org.apache.carbondata.core.path.CarbonTablePath;
+import org.apache.carbondata.core.atomic.AtomicFileOperations;
+import org.apache.carbondata.core.atomic.AtomicFileOperationsImpl;
+import org.apache.carbondata.core.atomic.FileWriteOperation;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastorage.impl.FileFactory;
-import org.apache.carbondata.core.load.LoadMetadataDetails;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.path.CarbonStorePath;
+import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.reader.CarbonDeleteFilesDataReader;
import org.apache.carbondata.core.update.CarbonUpdateUtil;
import org.apache.carbondata.core.update.SegmentUpdateDetails;
import org.apache.carbondata.core.update.TupleIdEnum;
import org.apache.carbondata.core.update.data.UpdateVO;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.atomic.AtomicFileOperations;
-import org.apache.carbondata.core.atomic.AtomicFileOperationsImpl;
-import org.apache.carbondata.core.atomic.FileWriteOperation;
import org.apache.carbondata.core.updatestatus.locks.CarbonLockFactory;
import org.apache.carbondata.core.updatestatus.locks.ICarbonLock;
import org.apache.carbondata.core.updatestatus.locks.LockUsage;
+import org.apache.carbondata.core.util.CarbonUtil;
import com.google.gson.Gson;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockFactory.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockFactory.java
index 0828ada..228d59c 100644
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockFactory.java
@@ -18,8 +18,8 @@
*/
package org.apache.carbondata.core.updatestatus.locks;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.util.CarbonProperties;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/HdfsFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/HdfsFileLock.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/HdfsFileLock.java
index 1d1de81..ca741c1 100644
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/HdfsFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/HdfsFileLock.java
@@ -23,10 +23,10 @@ import java.io.IOException;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLock.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLock.java
index cdee312..eeca76c 100644
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLock.java
@@ -26,9 +26,9 @@ import java.nio.channels.OverlappingFileLockException;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
/**
* This class handles the file locking in the local file system.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZooKeeperLocking.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZooKeeperLocking.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZooKeeperLocking.java
index 2d742cf..5e750b5 100644
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZooKeeperLocking.java
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZooKeeperLocking.java
@@ -24,8 +24,8 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.zookeeper.CreateMode;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
index fa8410a..78dd3ad 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
@@ -24,9 +24,13 @@ import java.util.ArrayList;
import java.util.BitSet;
import java.util.List;
-import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.BlockInfo;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.metadata.Encoding;
+import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.metadata.blocklet.SegmentInfo;
import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk;
@@ -34,11 +38,7 @@ import org.apache.carbondata.core.metadata.blocklet.datachunk.PresenceMeta;
import org.apache.carbondata.core.metadata.blocklet.index.BlockletBTreeIndex;
import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
import org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex;
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.core.reader.CarbonIndexFileReader;
import org.apache.carbondata.format.BlockIndex;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
index 79d5b7b..35cca0f 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
@@ -30,14 +30,14 @@ import java.util.Set;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.metadata.index.BlockIndexInfo;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.metadata.BlockletInfoColumnar;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
+import org.apache.carbondata.core.metadata.index.BlockIndexInfo;
import org.apache.carbondata.format.BlockIndex;
import org.apache.carbondata.format.BlockletBTreeIndex;
import org.apache.carbondata.format.BlockletIndex;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index 95d099a..281a13e 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -27,8 +27,8 @@ import java.util.Properties;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
public final class CarbonProperties {
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/util/CarbonTimeStatisticsFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonTimeStatisticsFactory.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonTimeStatisticsFactory.java
index 1bf2878..90b2317 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonTimeStatisticsFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonTimeStatisticsFactory.java
@@ -17,12 +17,12 @@
package org.apache.carbondata.core.util;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.stats.DriverQueryStatisticsRecorderDummy;
import org.apache.carbondata.core.stats.DriverQueryStatisticsRecorderImpl;
import org.apache.carbondata.core.stats.QueryStatisticsRecorder;
import org.apache.carbondata.core.stats.QueryStatisticsRecorderDummy;
import org.apache.carbondata.core.stats.QueryStatisticsRecorderImpl;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
public class CarbonTimeStatisticsFactory {
private static String loadStatisticsInstanceType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 34f1ac9..7e59345 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -40,19 +40,10 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
-import org.apache.carbondata.core.datastore.block.AbstractIndex;
-import org.apache.carbondata.core.datastore.block.TableBlockInfo;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
-import org.apache.carbondata.core.path.CarbonStorePath;
-import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.FileHolder;
import org.apache.carbondata.core.datastorage.columnar.ColumnGroupModel;
@@ -61,7 +52,13 @@ import org.apache.carbondata.core.datastorage.compression.MeasureMetaDataModel;
import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.block.AbstractIndex;
+import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
@@ -69,13 +66,16 @@ import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.path.CarbonStorePath;
+import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.reader.ThriftReader;
import org.apache.carbondata.core.reader.ThriftReader.TBaseCreator;
+import org.apache.carbondata.core.scan.model.QueryDimension;
+import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.core.service.PathService;
import org.apache.carbondata.core.update.data.UpdateVO;
import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager;
import org.apache.carbondata.format.DataChunk2;
-import org.apache.carbondata.core.scan.model.QueryDimension;
import org.apache.commons.lang.ArrayUtils;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
index eadac33..e590a5c 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
@@ -23,15 +23,15 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
+import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk;
import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.datastorage.FileHolder;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.reader.CarbonFooterReader;
import org.apache.carbondata.format.FileFooter;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
index 0808cea..0d71d7e 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
@@ -22,8 +22,8 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index a4aa931..c636555 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -31,10 +31,10 @@ import java.util.Map;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.spark.unsafe.types.UTF8String;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
index e024653..e368788 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
@@ -25,18 +25,18 @@ import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
-import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
-import org.apache.carbondata.core.metadata.ColumnIdentifier;
-import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
+import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk;
import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader;
import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReaderImpl;
+import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.core.service.PathService;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.format.ColumnDictionaryChunk;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
index 1d37b5b..89a1c52 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
@@ -23,15 +23,15 @@ import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
-import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
-import org.apache.carbondata.core.metadata.ColumnIdentifier;
-import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
+import org.apache.carbondata.core.path.CarbonTablePath;
+import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.core.service.PathService;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparator.java b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparator.java
index aa35565..1d4f366 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparator.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparator.java
@@ -25,8 +25,8 @@ import java.util.List;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryChunksWrapper;
-import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.DataType;
import org.apache.commons.lang.ArrayUtils;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java
index 6ec89f5..03604c2 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortModel.java
@@ -22,8 +22,8 @@ import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
-import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.util.CarbonProperties;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/test/java/org/apache/carbondata/core/load/LoadMetadataDetailsUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/load/LoadMetadataDetailsUnitTest.java b/core/src/test/java/org/apache/carbondata/core/load/LoadMetadataDetailsUnitTest.java
deleted file mode 100644
index 0dbac8f..0000000
--- a/core/src/test/java/org/apache/carbondata/core/load/LoadMetadataDetailsUnitTest.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.load;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-
-import static junit.framework.Assert.*;
-
-public class LoadMetadataDetailsUnitTest {
-
- private LoadMetadataDetails loadMetadataDetails;
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(LoadMetadataDetails.class.getName());
-
- @Before public void setup() {
- loadMetadataDetails = new LoadMetadataDetails();
- }
-
- /**
- * This method will test Hashcode which will return 31 if we don't set loadName.
- *
- * @throws Exception
- */
-
- @Test public void testHashCodeLoadNameNull() throws Exception {
- int expected_result = 31;
- int data = loadMetadataDetails.hashCode();
- assertEquals(expected_result, data);
- }
-
- @Test public void testHashCodeValueInLoadName() throws Exception {
- loadMetadataDetails.setLoadName("test");
- int data = loadMetadataDetails.hashCode();
- assertNotSame(31, data);
- }
-
- @Test public void testEqualsObjectIsNotLoadMetadataDetails() throws Exception {
- Object obj = new Object();
- boolean result = loadMetadataDetails.equals(obj);
- assertEquals(false, result);
- }
-
- @Test public void testEqualsObjectIsNull() throws Exception {
- boolean result = loadMetadataDetails.equals(new Object());
- assertEquals(false, result);
- }
-
- @Test public void testEqualsObjectIsLoadMetadataDetailsWithoutLoadName() throws Exception {
- LoadMetadataDetails obj = new LoadMetadataDetails();
- boolean result = loadMetadataDetails.equals(obj);
- assertEquals(true, result);
- }
-
- @Test public void testEqualsObjectIsLoadMetadataDetails() throws Exception {
- loadMetadataDetails.setLoadName("test");
- LoadMetadataDetails obj = new LoadMetadataDetails();
- boolean result = loadMetadataDetails.equals(obj);
- assertEquals(false, result);
- }
-
- @Test public void testEqualsObjectIsLoadMetadataDetailsLoadNameNull() throws Exception {
- LoadMetadataDetails obj = new LoadMetadataDetails();
- obj.setLoadName("test");
- boolean result = loadMetadataDetails.equals(obj);
- assertEquals(false, result);
- }
-
- @Test public void testEqualsObjectIsLoadMetadataDetailsLoadNameEqualsObjectLoadName()
- throws Exception {
- loadMetadataDetails.setLoadName("test");
- LoadMetadataDetails obj = new LoadMetadataDetails();
- obj.setLoadName("test");
- boolean result = loadMetadataDetails.equals(obj);
- assertEquals(true, result);
- }
-
- @Test public void testGetTimeStampWithDate() throws Exception {
- String date = "01-01-2016 00:00:00:000";
- long longVal = loadMetadataDetails.getTimeStamp(date);
- loadMetadataDetails.setLoadStartTime(longVal);
- Long expected_result = getTime(date);
- Long result = loadMetadataDetails.getLoadStartTime();
- assertEquals(expected_result, result);
- }
-
- public static Long getTime(String date) {
- SimpleDateFormat simpleDateFormat = new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP_MILLIS);
- try {
- return simpleDateFormat.parse(date).getTime() * 1000;
- } catch (ParseException e) {
- LOGGER.error("Error while parsing " + date + " " + e.getMessage());
- return null;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/test/java/org/apache/carbondata/core/updatestatus/LoadMetadataDetailsUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/updatestatus/LoadMetadataDetailsUnitTest.java b/core/src/test/java/org/apache/carbondata/core/updatestatus/LoadMetadataDetailsUnitTest.java
new file mode 100644
index 0000000..a4b4708
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/updatestatus/LoadMetadataDetailsUnitTest.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.updatestatus;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertNotSame;
+
+public class LoadMetadataDetailsUnitTest {
+
+ private LoadMetadataDetails loadMetadataDetails;
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(LoadMetadataDetails.class.getName());
+
+ @Before public void setup() {
+ loadMetadataDetails = new LoadMetadataDetails();
+ }
+
+ /**
+ * This method will test Hashcode which will return 31 if we don't set loadName.
+ *
+ * @throws Exception
+ */
+
+ @Test public void testHashCodeLoadNameNull() throws Exception {
+ int expected_result = 31;
+ int data = loadMetadataDetails.hashCode();
+ assertEquals(expected_result, data);
+ }
+
+ @Test public void testHashCodeValueInLoadName() throws Exception {
+ loadMetadataDetails.setLoadName("test");
+ int data = loadMetadataDetails.hashCode();
+ assertNotSame(31, data);
+ }
+
+ @Test public void testEqualsObjectIsNotLoadMetadataDetails() throws Exception {
+ Object obj = new Object();
+ boolean result = loadMetadataDetails.equals(obj);
+ assertEquals(false, result);
+ }
+
+ @Test public void testEqualsObjectIsNull() throws Exception {
+ boolean result = loadMetadataDetails.equals(new Object());
+ assertEquals(false, result);
+ }
+
+ @Test public void testEqualsObjectIsLoadMetadataDetailsWithoutLoadName() throws Exception {
+ LoadMetadataDetails obj = new LoadMetadataDetails();
+ boolean result = loadMetadataDetails.equals(obj);
+ assertEquals(true, result);
+ }
+
+ @Test public void testEqualsObjectIsLoadMetadataDetails() throws Exception {
+ loadMetadataDetails.setLoadName("test");
+ LoadMetadataDetails obj = new LoadMetadataDetails();
+ boolean result = loadMetadataDetails.equals(obj);
+ assertEquals(false, result);
+ }
+
+ @Test public void testEqualsObjectIsLoadMetadataDetailsLoadNameNull() throws Exception {
+ LoadMetadataDetails obj = new LoadMetadataDetails();
+ obj.setLoadName("test");
+ boolean result = loadMetadataDetails.equals(obj);
+ assertEquals(false, result);
+ }
+
+ @Test public void testEqualsObjectIsLoadMetadataDetailsLoadNameEqualsObjectLoadName()
+ throws Exception {
+ loadMetadataDetails.setLoadName("test");
+ LoadMetadataDetails obj = new LoadMetadataDetails();
+ obj.setLoadName("test");
+ boolean result = loadMetadataDetails.equals(obj);
+ assertEquals(true, result);
+ }
+
+ @Test public void testGetTimeStampWithDate() throws Exception {
+ String date = "01-01-2016 00:00:00:000";
+ long longVal = loadMetadataDetails.getTimeStamp(date);
+ loadMetadataDetails.setLoadStartTime(longVal);
+ Long expected_result = getTime(date);
+ Long result = loadMetadataDetails.getLoadStartTime();
+ assertEquals(expected_result, result);
+ }
+
+ public static Long getTime(String date) {
+ SimpleDateFormat simpleDateFormat = new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP_MILLIS);
+ try {
+ return simpleDateFormat.parse(date).getTime() * 1000;
+ } catch (ParseException e) {
+ LOGGER.error("Error while parsing " + date + " " + e.getMessage());
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
index 384e7c6..9a6e269 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
@@ -48,8 +48,8 @@ import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.impl.FileFactory;
-import org.apache.carbondata.core.load.BlockDetails;
-import org.apache.carbondata.core.load.LoadMetadataDetails;
+import org.apache.carbondata.processing.csvreaderstep.BlockDetails;
+import org.apache.carbondata.core.updatestatus.LoadMetadataDetails;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.writer.CarbonDictionaryWriter;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
index 76e815f..e6922fe 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
@@ -21,7 +21,7 @@ package org.apache.carbondata.spark.testsuite.dataretention
import java.text.SimpleDateFormat
-import org.apache.carbondata.core.updatestatus.SegmentStatusManager
+import org.apache.carbondata.core.updatestatus.{LoadMetadataDetails, SegmentStatusManager}
import org.apache.carbondata.core.updatestatus.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
import org.apache.commons.lang3.time.DateUtils
import org.apache.spark.sql.Row
@@ -31,7 +31,6 @@ import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.core.CarbonTableIdentifier
import org.apache.carbondata.core.path.CarbonStorePath
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.load.LoadMetadataDetails
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
index 9c80d7c..75bc294 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
@@ -46,7 +46,7 @@ import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.datastorage.impl.FileFactory.FileType;
-import org.apache.carbondata.core.load.LoadMetadataDetails;
+import org.apache.carbondata.core.updatestatus.LoadMetadataDetails;
import org.apache.carbondata.core.update.CarbonUpdateUtil;
import org.apache.carbondata.core.updatestatus.SegmentStatusManager;
import org.apache.carbondata.core.util.CarbonProperties;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java
index 3c18f64..470ab9a 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java
@@ -42,7 +42,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastorage.impl.FileFactory;
-import org.apache.carbondata.core.load.LoadMetadataDetails;
+import org.apache.carbondata.core.updatestatus.LoadMetadataDetails;
import org.apache.carbondata.core.update.CarbonUpdateUtil;
public final class DeleteLoadFolders {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
index 7ef4cec..82d3f0e 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
@@ -36,7 +36,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastorage.impl.FileFactory;
-import org.apache.carbondata.core.load.LoadMetadataDetails;
+import org.apache.carbondata.core.updatestatus.LoadMetadataDetails;
import org.apache.carbondata.core.reader.CarbonDeleteFilesDataReader;
import org.apache.carbondata.core.update.CarbonUpdateUtil;
import org.apache.carbondata.core.update.DeleteDeltaBlockDetails;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/CarbonQueryUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/CarbonQueryUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/CarbonQueryUtil.java
index ffce6cb..8db32a2 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/CarbonQueryUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/CarbonQueryUtil.java
@@ -26,7 +26,7 @@ import java.util.List;
import java.util.Map;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.load.LoadMetadataDetails;
+import org.apache.carbondata.core.updatestatus.LoadMetadataDetails;
import org.apache.carbondata.core.scan.model.CarbonQueryPlan;
import org.apache.carbondata.spark.partition.api.Partition;
import org.apache.carbondata.spark.partition.api.impl.DefaultLoadBalancer;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/LoadMetadataUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/LoadMetadataUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/LoadMetadataUtil.java
index 9b67fe3..e8d4dde 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/LoadMetadataUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/LoadMetadataUtil.java
@@ -31,7 +31,7 @@ package org.apache.carbondata.spark.util;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.load.LoadMetadataDetails;
+import org.apache.carbondata.core.updatestatus.LoadMetadataDetails;
import org.apache.carbondata.core.updatestatus.SegmentStatusManager;
public final class LoadMetadataUtil {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/KeyVal.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/KeyVal.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/KeyVal.scala
index ad621bc..fbf0fd0 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/KeyVal.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/KeyVal.scala
@@ -27,8 +27,8 @@ package org.apache.carbondata.spark
import org.apache.spark.sql.execution.command.ExecutionErrors
-import org.apache.carbondata.core.load.LoadMetadataDetails
import org.apache.carbondata.core.update.SegmentUpdateDetails
+import org.apache.carbondata.core.updatestatus.LoadMetadataDetails
trait Value[V] extends Serializable {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala
index 4392775..796ef7d 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala
@@ -36,15 +36,15 @@ import org.apache.spark.util.SparkUtil
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.common.logging.impl.StandardLogService
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.load.{BlockDetails, LoadMetadataDetails}
+import org.apache.carbondata.core.updatestatus.LoadMetadataDetails
import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory}
import org.apache.carbondata.processing.constants.DataProcessorConstants
-import org.apache.carbondata.processing.csvreaderstep.{JavaRddIterator, RddInputUtils}
+import org.apache.carbondata.processing.csvreaderstep.{BlockDetails, JavaRddIterator, RddInputUtils}
import org.apache.carbondata.processing.etl.DataLoadingException
import org.apache.carbondata.processing.graphgenerator.GraphGenerator
import org.apache.carbondata.processing.model.CarbonLoadModel
import org.apache.carbondata.spark.DataLoadResult
-import org.apache.carbondata.spark.load.{_}
+import org.apache.carbondata.spark.load._
import org.apache.carbondata.spark.splits.TableSplit
import org.apache.carbondata.spark.util.CarbonQueryUtil
import org.apache.carbondata.spark.util.CarbonScalaUtil
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeleteLoadByDateRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeleteLoadByDateRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeleteLoadByDateRDD.scala
index 33f453b..268d52c 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeleteLoadByDateRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeleteLoadByDateRDD.scala
@@ -23,7 +23,7 @@ import org.apache.spark.{Partition, SparkContext, TaskContext}
import org.apache.spark.rdd.RDD
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.load.LoadMetadataDetails
+import org.apache.carbondata.core.updatestatus.LoadMetadataDetails
import org.apache.carbondata.spark.DeletedLoadResult
import org.apache.carbondata.spark.util.CarbonQueryUtil
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
index de57639..e4d0671 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
@@ -30,9 +30,8 @@ import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.metadata.{CarbonMetadata, CarbonTableIdentifier}
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.load.LoadMetadataDetails
import org.apache.carbondata.core.update.CarbonUpdateUtil
-import org.apache.carbondata.core.updatestatus.SegmentStatusManager
+import org.apache.carbondata.core.updatestatus.{LoadMetadataDetails, SegmentStatusManager}
import org.apache.carbondata.core.updatestatus.locks.{CarbonLockFactory, CarbonLockUtil, LockUsage}
import org.apache.carbondata.processing.model.{CarbonDataLoadSchema, CarbonLoadModel}
import org.apache.carbondata.spark._
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
index 497df75..e63f904 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
@@ -40,11 +40,11 @@ import org.apache.carbondata.common.CarbonIterator
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.common.logging.impl.StandardLogService
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.load.{BlockDetails, LoadMetadataDetails}
+import org.apache.carbondata.core.updatestatus.LoadMetadataDetails
import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory}
import org.apache.carbondata.hadoop.csv.CSVInputFormat
import org.apache.carbondata.hadoop.csv.recorditerator.RecordReaderIterator
-import org.apache.carbondata.processing.csvreaderstep.JavaRddIterator
+import org.apache.carbondata.processing.csvreaderstep.{BlockDetails, JavaRddIterator}
import org.apache.carbondata.processing.model.CarbonLoadModel
import org.apache.carbondata.processing.newflow.DataLoadExecutor
import org.apache.carbondata.processing.newflow.exception.BadRecordFoundException
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 406c94f..ba468f8 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -32,9 +32,8 @@ import org.apache.carbondata.core.metadata.schema.{BucketingInfo, SchemaEvolutio
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo, TableSchema}
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.load.LoadMetadataDetails
import org.apache.carbondata.core.service.CarbonCommonFactory
-import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager
+import org.apache.carbondata.core.updatestatus.{LoadMetadataDetails, SegmentUpdateStatusManager}
import org.apache.carbondata.processing.model.CarbonLoadModel
import org.apache.carbondata.spark.CarbonSparkFactory
import org.apache.carbondata.spark.load.FailureCauses
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/integration/spark-common/src/main/scala/org/apache/spark/util/SparkUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/util/SparkUtil.scala b/integration/spark-common/src/main/scala/org/apache/spark/util/SparkUtil.scala
index e2f4c3e..7c6665c 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/util/SparkUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/util/SparkUtil.scala
@@ -23,7 +23,7 @@ import org.apache.hadoop.mapreduce.lib.input.{FileInputFormat, FileSplit}
import org.apache.spark.{SparkContext, TaskContext}
import org.apache.spark.rdd.{NewHadoopPartition, NewHadoopRDD}
-import org.apache.carbondata.core.load.BlockDetails
+import org.apache.carbondata.processing.csvreaderstep.BlockDetails
/*
* this object use to handle file splits
[21/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/ResolvedFilterInfoVisitorIntf.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/ResolvedFilterInfoVisitorIntf.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/ResolvedFilterInfoVisitorIntf.java
new file mode 100644
index 0000000..defc0d4
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/ResolvedFilterInfoVisitorIntf.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.filter.resolver.resolverinfo.visitor;
+
+import java.io.IOException;
+
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.resolver.metadata.FilterResolverMetadata;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+
+public interface ResolvedFilterInfoVisitorIntf {
+
+ /**
+ * Visitor pattern is been used in this scenario inorder to populate the
+ * dimColResolvedFilterInfo visitable object with filter member values based
+ * on the visitor type, currently there 3 types of visitors custom,direct
+ * and no dictionary, all types of visitor populate the visitable instance
+ * as per its buisness logic which is different for all the visitors.
+ *
+ * @param visitableObj
+ * @param metadata
+ * @throws FilterUnsupportedException
+ */
+ void populateFilterResolvedInfo(DimColumnResolvedFilterInfo visitableObj,
+ FilterResolverMetadata metadata) throws FilterUnsupportedException, IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/model/CarbonQueryPlan.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/CarbonQueryPlan.java b/core/src/main/java/org/apache/carbondata/core/scan/model/CarbonQueryPlan.java
new file mode 100644
index 0000000..6b520ff
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/CarbonQueryPlan.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ *
+ */
+package org.apache.carbondata.core.scan.model;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.scan.expression.Expression;
+
+/**
+ * This class contains all the logical information about the query like dimensions,measures,
+ * sort order, topN etc..
+ */
+public class CarbonQueryPlan implements Serializable {
+ /**
+ *
+ */
+ private static final long serialVersionUID = -9036044826928017164L;
+
+ /**
+ * Database name
+ */
+ private String databaseName;
+
+ /**
+ * Table name
+ */
+ private String tableName;
+
+ /**
+ * List of dimensions.
+ * Ex : select employee_name,department_name,sum(salary) from employee, then employee_name
+ * and department_name are dimensions
+ * If there is no dimensions asked in query then it would be remained as empty.
+ */
+ private List<QueryDimension> dimensions =
+ new ArrayList<QueryDimension>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+
+ /**
+ * List of measures.
+ * Ex : select employee_name,department_name,sum(salary) from employee, then sum(salary)
+ * would be measure.
+ * If there is no dimensions asked in query then it would be remained as empty.
+ */
+ private List<QueryMeasure> measures =
+ new ArrayList<QueryMeasure>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+
+ /**
+ * expression
+ */
+ private Expression expression;
+
+ /**
+ * queryId
+ */
+ private String queryId;
+
+ /**
+ * If it is raw detail query, no need to aggregate in backend. And it returns with dictionary data
+ * with out decoding.
+ */
+ private boolean rawDetailQuery;
+
+ /**
+ * Constructor created with database name and table name.
+ *
+ * @param databaseName
+ * @param tableName
+ */
+ public CarbonQueryPlan(String databaseName, String tableName) {
+ this.tableName = tableName;
+ this.databaseName = databaseName;
+ }
+
+ /**
+ * @return the dimensions
+ */
+ public List<QueryDimension> getDimensions() {
+ return dimensions;
+ }
+
+ public void addDimension(QueryDimension dimension) {
+ this.dimensions.add(dimension);
+ }
+
+ /**
+ * @return the measures
+ */
+ public List<QueryMeasure> getMeasures() {
+ return measures;
+ }
+
+ public void addMeasure(QueryMeasure measure) {
+ this.measures.add(measure);
+ }
+
+ public Expression getFilterExpression() {
+ return expression;
+ }
+
+ public void setFilterExpression(Expression expression) {
+ this.expression = expression;
+ }
+
+ /**
+ * @return the databaseName
+ */
+ public String getDatabaseName() {
+ return databaseName;
+ }
+
+ /**
+ * @return the tableName
+ */
+ public String getTableName() {
+ return tableName;
+ }
+
+ public String getQueryId() {
+ return queryId;
+ }
+
+ public void setQueryId(String queryId) {
+ this.queryId = queryId;
+ }
+
+ public boolean isRawDetailQuery() {
+ return rawDetailQuery;
+ }
+
+ public void setRawDetailQuery(boolean rawDetailQuery) {
+ this.rawDetailQuery = rawDetailQuery;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/model/QueryColumn.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryColumn.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryColumn.java
new file mode 100644
index 0000000..e1cc825
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryColumn.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.model;
+
+import java.io.Serializable;
+
+/**
+ * query column which will have information about column
+ */
+public class QueryColumn implements Serializable {
+
+ /**
+ * serialVersionUID
+ */
+ private static final long serialVersionUID = -4222306600480181084L;
+
+ /**
+ * name of the column
+ */
+ protected String columnName;
+
+ /**
+ * query order in which result of the query will be send
+ */
+ private int queryOrder;
+
+ public QueryColumn(String columnName) {
+ this.columnName = columnName;
+ }
+
+ /**
+ * @return the columnName
+ */
+ public String getColumnName() {
+ return columnName;
+ }
+
+ /**
+ * @return the queryOrder
+ */
+ public int getQueryOrder() {
+ return queryOrder;
+ }
+
+ /**
+ * @param queryOrder the queryOrder to set
+ */
+ public void setQueryOrder(int queryOrder) {
+ this.queryOrder = queryOrder;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/model/QueryDimension.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryDimension.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryDimension.java
new file mode 100644
index 0000000..2d84ee1
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryDimension.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.model;
+
+import java.io.Serializable;
+
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+
+/**
+ * query plan dimension which will holds the information about the query plan dimension
+ * this is done to avoid heavy object serialization
+ */
+public class QueryDimension extends QueryColumn implements Serializable {
+
+ /**
+ * serialVersionUID
+ */
+ private static final long serialVersionUID = -8492704093776645651L;
+ /**
+ * actual dimension column
+ */
+ private transient CarbonDimension dimension;
+
+ public QueryDimension(String columnName) {
+ super(columnName);
+ }
+
+ /**
+ * @return the dimension
+ */
+ public CarbonDimension getDimension() {
+ return dimension;
+ }
+
+ /**
+ * @param dimension the dimension to set
+ */
+ public void setDimension(CarbonDimension dimension) {
+ this.dimension = dimension;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/model/QueryMeasure.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryMeasure.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryMeasure.java
new file mode 100644
index 0000000..3ca6010
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryMeasure.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.model;
+
+import java.io.Serializable;
+
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
+
+/**
+ * query plan measure, this class will holds the information
+ * about measure present in the query, this is done to avoid the serialization
+ * of the heavy object
+ */
+public class QueryMeasure extends QueryColumn implements Serializable {
+
+ /**
+ * serialVersionUID
+ */
+ private static final long serialVersionUID = 1035512411375495414L;
+
+ /**
+ * actual carbon measure object
+ */
+ private transient CarbonMeasure measure;
+
+ public QueryMeasure(String columnName) {
+ super(columnName);
+ }
+
+ /**
+ * @return the measure
+ */
+ public CarbonMeasure getMeasure() {
+ return measure;
+ }
+
+ /**
+ * @param measure the measure to set
+ */
+ public void setMeasure(CarbonMeasure measure) {
+ this.measure = measure;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
new file mode 100644
index 0000000..1b295db
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
@@ -0,0 +1,365 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.model;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.core.cache.dictionary.Dictionary;
+import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
+import org.apache.carbondata.core.stats.QueryStatisticsRecorder;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.update.data.UpdateVO;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.UnknownExpression;
+import org.apache.carbondata.core.scan.expression.conditional.ConditionalExpression;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+
+/**
+ * Query model which will have all the detail
+ * about the query, This will be sent from driver to executor '
+ * This will be refereed to executing the query.
+ */
+public class QueryModel implements Serializable {
+
+ /**
+ * serialization version
+ */
+ private static final long serialVersionUID = -4674677234007089052L;
+ /**
+ * this will hold the information about the dictionary dimension
+ * which to
+ */
+ public transient Map<String, Dictionary> columnToDictionaryMapping;
+ /**
+ * list of dimension selected for in query
+ */
+ private List<QueryDimension> queryDimension;
+ /**
+ * list of measure selected in query
+ */
+ private List<QueryMeasure> queryMeasures;
+ /**
+ * query id
+ */
+ private String queryId;
+ /**
+ * filter tree
+ */
+ private FilterResolverIntf filterExpressionResolverTree;
+
+ /**
+ * table block information in which query will be executed
+ */
+ private List<TableBlockInfo> tableBlockInfos;
+ /**
+ * absolute table identifier
+ */
+ private AbsoluteTableIdentifier absoluteTableIdentifier;
+ /**
+ * To handle most of the computation in query engines like spark and hive, carbon should give
+ * raw detailed records to it.
+ */
+ private boolean forcedDetailRawQuery;
+ /**
+ * table on which query will be executed
+ * TODO need to remove this ad pass only the path
+ * and carbon metadata will load the table from metadata file
+ */
+ private CarbonTable table;
+
+ private QueryStatisticsRecorder statisticsRecorder;
+
+ private boolean vectorReader;
+
+ /**
+ * Invalid table blocks, which need to be removed from
+ * memory, invalid blocks can be segment which are deleted
+ * or compacted
+ */
+ private List<String> invalidSegmentIds;
+ private Map<String, UpdateVO> invalidSegmentBlockIdMap =
+ new HashMap<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+
+ public QueryModel() {
+ tableBlockInfos = new ArrayList<TableBlockInfo>();
+ queryDimension = new ArrayList<QueryDimension>();
+ queryMeasures = new ArrayList<QueryMeasure>();
+ invalidSegmentIds = new ArrayList<>();
+ }
+
+ public static QueryModel createModel(AbsoluteTableIdentifier absoluteTableIdentifier,
+ CarbonQueryPlan queryPlan, CarbonTable carbonTable) {
+ QueryModel queryModel = new QueryModel();
+ String factTableName = carbonTable.getFactTableName();
+ queryModel.setAbsoluteTableIdentifier(absoluteTableIdentifier);
+
+ fillQueryModel(queryPlan, carbonTable, queryModel, factTableName);
+
+ queryModel.setForcedDetailRawQuery(queryPlan.isRawDetailQuery());
+ queryModel.setQueryId(queryPlan.getQueryId());
+ return queryModel;
+ }
+
+ private static void fillQueryModel(CarbonQueryPlan queryPlan, CarbonTable carbonTable,
+ QueryModel queryModel, String factTableName) {
+ queryModel.setAbsoluteTableIdentifier(carbonTable.getAbsoluteTableIdentifier());
+ queryModel.setQueryDimension(queryPlan.getDimensions());
+ queryModel.setQueryMeasures(queryPlan.getMeasures());
+ if (null != queryPlan.getFilterExpression()) {
+ processFilterExpression(queryPlan.getFilterExpression(),
+ carbonTable.getDimensionByTableName(factTableName),
+ carbonTable.getMeasureByTableName(factTableName));
+ }
+ //TODO need to remove this code, and executor will load the table
+ // from file metadata
+ queryModel.setTable(carbonTable);
+ }
+
+ public static void processFilterExpression(Expression filterExpression,
+ List<CarbonDimension> dimensions, List<CarbonMeasure> measures) {
+ if (null != filterExpression) {
+ if (null != filterExpression.getChildren() && filterExpression.getChildren().size() == 0) {
+ if (filterExpression instanceof ConditionalExpression) {
+ List<ColumnExpression> listOfCol =
+ ((ConditionalExpression) filterExpression).getColumnList();
+ for (ColumnExpression expression : listOfCol) {
+ setDimAndMsrColumnNode(dimensions, measures, expression);
+ }
+ }
+ }
+ for (Expression expression : filterExpression.getChildren()) {
+ if (expression instanceof ColumnExpression) {
+ setDimAndMsrColumnNode(dimensions, measures, (ColumnExpression) expression);
+ } else if (expression instanceof UnknownExpression) {
+ UnknownExpression exp = ((UnknownExpression) expression);
+ List<ColumnExpression> listOfColExpression = exp.getAllColumnList();
+ for (ColumnExpression col : listOfColExpression) {
+ setDimAndMsrColumnNode(dimensions, measures, col);
+ }
+ } else {
+ processFilterExpression(expression, dimensions, measures);
+ }
+ }
+ }
+
+ }
+
+ private static CarbonMeasure getCarbonMetadataMeasure(String name, List<CarbonMeasure> measures) {
+ for (CarbonMeasure measure : measures) {
+ if (measure.getColName().equalsIgnoreCase(name)) {
+ return measure;
+ }
+ }
+ return null;
+ }
+
+ private static void setDimAndMsrColumnNode(List<CarbonDimension> dimensions,
+ List<CarbonMeasure> measures, ColumnExpression col) {
+ CarbonDimension dim;
+ CarbonMeasure msr;
+ String columnName;
+ columnName = col.getColumnName();
+ dim = CarbonUtil.findDimension(dimensions, columnName);
+ col.setCarbonColumn(dim);
+ col.setDimension(dim);
+ col.setDimension(true);
+ if (null == dim) {
+ msr = getCarbonMetadataMeasure(columnName, measures);
+ col.setCarbonColumn(msr);
+ col.setDimension(false);
+ }
+ }
+
+ /**
+ * It gets the projection columns
+ */
+ public CarbonColumn[] getProjectionColumns() {
+ CarbonColumn[] carbonColumns =
+ new CarbonColumn[getQueryDimension().size() + getQueryMeasures().size()];
+ for (QueryDimension dimension : getQueryDimension()) {
+ carbonColumns[dimension.getQueryOrder()] = dimension.getDimension();
+ }
+ for (QueryMeasure msr : getQueryMeasures()) {
+ carbonColumns[msr.getQueryOrder()] = msr.getMeasure();
+ }
+ return carbonColumns;
+ }
+
+ /**
+ * @return the queryDimension
+ */
+ public List<QueryDimension> getQueryDimension() {
+ return queryDimension;
+ }
+
+ /**
+ * @param queryDimension the queryDimension to set
+ */
+ public void setQueryDimension(List<QueryDimension> queryDimension) {
+ this.queryDimension = queryDimension;
+ }
+
+ /**
+ * @return the queryMeasures
+ */
+ public List<QueryMeasure> getQueryMeasures() {
+ return queryMeasures;
+ }
+
+ /**
+ * @param queryMeasures the queryMeasures to set
+ */
+ public void setQueryMeasures(List<QueryMeasure> queryMeasures) {
+ this.queryMeasures = queryMeasures;
+ }
+
+ /**
+ * @return the queryId
+ */
+ public String getQueryId() {
+ return queryId;
+ }
+
+ /**
+ * @param queryId the queryId to set
+ */
+ public void setQueryId(String queryId) {
+ this.queryId = queryId;
+ }
+
+ /**
+ * @return the tableBlockInfos
+ */
+ public List<TableBlockInfo> getTableBlockInfos() {
+ return tableBlockInfos;
+ }
+
+ /**
+ * @param tableBlockInfos the tableBlockInfos to set
+ */
+ public void setTableBlockInfos(List<TableBlockInfo> tableBlockInfos) {
+ this.tableBlockInfos = tableBlockInfos;
+ }
+
+ /**
+ * @return the filterEvaluatorTree
+ */
+ public FilterResolverIntf getFilterExpressionResolverTree() {
+ return filterExpressionResolverTree;
+ }
+
+ public void setFilterExpressionResolverTree(FilterResolverIntf filterExpressionResolverTree) {
+ this.filterExpressionResolverTree = filterExpressionResolverTree;
+ }
+
+ /**
+ * @return the absoluteTableIdentifier
+ */
+ public AbsoluteTableIdentifier getAbsoluteTableIdentifier() {
+ return absoluteTableIdentifier;
+ }
+
+ /**
+ * @param absoluteTableIdentifier the absoluteTableIdentifier to set
+ */
+ public void setAbsoluteTableIdentifier(AbsoluteTableIdentifier absoluteTableIdentifier) {
+ this.absoluteTableIdentifier = absoluteTableIdentifier;
+ }
+
+ /**
+ * @return the table
+ */
+ public CarbonTable getTable() {
+ return table;
+ }
+
+ /**
+ * @param table the table to set
+ */
+ public void setTable(CarbonTable table) {
+ this.table = table;
+ }
+
+ public boolean isForcedDetailRawQuery() {
+ return forcedDetailRawQuery;
+ }
+
+ public void setForcedDetailRawQuery(boolean forcedDetailRawQuery) {
+ this.forcedDetailRawQuery = forcedDetailRawQuery;
+ }
+
+ /**
+ * @return
+ */
+ public Map<String, Dictionary> getColumnToDictionaryMapping() {
+ return columnToDictionaryMapping;
+ }
+
+ /**
+ * @param columnToDictionaryMapping
+ */
+ public void setColumnToDictionaryMapping(Map<String, Dictionary> columnToDictionaryMapping) {
+ this.columnToDictionaryMapping = columnToDictionaryMapping;
+ }
+
+ public QueryStatisticsRecorder getStatisticsRecorder() {
+ return statisticsRecorder;
+ }
+
+ public void setStatisticsRecorder(QueryStatisticsRecorder statisticsRecorder) {
+ this.statisticsRecorder = statisticsRecorder;
+ }
+
+ public List<String> getInvalidSegmentIds() {
+ return invalidSegmentIds;
+ }
+
+ public void setInvalidSegmentIds(List<String> invalidSegmentIds) {
+ this.invalidSegmentIds = invalidSegmentIds;
+ }
+
+ public boolean isVectorReader() {
+ return vectorReader;
+ }
+
+ public void setVectorReader(boolean vectorReader) {
+ this.vectorReader = vectorReader;
+ }
+ public void setInvalidBlockForSegmentId(List<UpdateVO> invalidSegmentTimestampList) {
+ for (UpdateVO anUpdateVO : invalidSegmentTimestampList) {
+ this.invalidSegmentBlockIdMap.put(anUpdateVO.getSegmentId(), anUpdateVO);
+ }
+ }
+
+ public Map<String,UpdateVO> getInvalidBlockVOForSegmentId() {
+ return invalidSegmentBlockIdMap;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
new file mode 100644
index 0000000..dbbb6dc
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.processor;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.stats.QueryStatisticsModel;
+import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.scan.collector.ScannedResultCollector;
+import org.apache.carbondata.core.scan.collector.impl.DictionaryBasedResultCollector;
+import org.apache.carbondata.core.scan.collector.impl.DictionaryBasedVectorResultCollector;
+import org.apache.carbondata.core.scan.collector.impl.RawBasedResultCollector;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
+import org.apache.carbondata.core.scan.scanner.BlockletScanner;
+import org.apache.carbondata.core.scan.scanner.impl.FilterScanner;
+import org.apache.carbondata.core.scan.scanner.impl.NonFilterScanner;
+
+/**
+ * This abstract class provides a skeletal implementation of the
+ * Block iterator.
+ */
+public abstract class AbstractDataBlockIterator extends CarbonIterator<List<Object[]>> {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(AbstractDataBlockIterator.class.getName());
+
+ /**
+ * iterator which will be used to iterate over data blocks
+ */
+ protected CarbonIterator<DataRefNode> dataBlockIterator;
+
+ /**
+ * result collector which will be used to aggregate the scanned result
+ */
+ protected ScannedResultCollector scannerResultAggregator;
+
+ /**
+ * processor which will be used to process the block processing can be
+ * filter processing or non filter processing
+ */
+ protected BlockletScanner blockletScanner;
+
+ /**
+ * to hold the data block
+ */
+ protected BlocksChunkHolder blocksChunkHolder;
+
+ /**
+ * batch size of result
+ */
+ protected int batchSize;
+
+ protected AbstractScannedResult scannedResult;
+
+ public AbstractDataBlockIterator(BlockExecutionInfo blockExecutionInfo,
+ FileHolder fileReader, int batchSize, QueryStatisticsModel queryStatisticsModel) {
+ dataBlockIterator = new BlockletIterator(blockExecutionInfo.getFirstDataBlock(),
+ blockExecutionInfo.getNumberOfBlockToScan());
+ blocksChunkHolder = new BlocksChunkHolder(blockExecutionInfo.getTotalNumberDimensionBlock(),
+ blockExecutionInfo.getTotalNumberOfMeasureBlock());
+ blocksChunkHolder.setFileReader(fileReader);
+
+ if (blockExecutionInfo.getFilterExecuterTree() != null) {
+ blockletScanner = new FilterScanner(blockExecutionInfo, queryStatisticsModel);
+ } else {
+ blockletScanner = new NonFilterScanner(blockExecutionInfo, queryStatisticsModel);
+ }
+ if (blockExecutionInfo.isRawRecordDetailQuery()) {
+ LOGGER.info("Row based raw collector is used to scan and collect the data");
+ this.scannerResultAggregator =
+ new RawBasedResultCollector(blockExecutionInfo);
+ } else if (blockExecutionInfo.isVectorBatchCollector()) {
+ LOGGER.info("Vector based dictionary collector is used to scan and collect the data");
+ this.scannerResultAggregator =
+ new DictionaryBasedVectorResultCollector(blockExecutionInfo);
+ } else {
+ LOGGER.info("Row based dictionary collector is used to scan and collect the data");
+ this.scannerResultAggregator =
+ new DictionaryBasedResultCollector(blockExecutionInfo);
+ }
+ this.batchSize = batchSize;
+ }
+
+ public boolean hasNext() {
+ if (scannedResult != null && scannedResult.hasNext()) {
+ return true;
+ } else {
+ return dataBlockIterator.hasNext();
+ }
+ }
+
+ protected boolean updateScanner() {
+ try {
+ if (scannedResult != null && scannedResult.hasNext()) {
+ return true;
+ } else {
+ scannedResult = getNextScannedResult();
+ while (scannedResult != null) {
+ if (scannedResult.hasNext()) {
+ return true;
+ }
+ scannedResult = getNextScannedResult();
+ }
+ return false;
+ }
+ } catch (IOException | FilterUnsupportedException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ private AbstractScannedResult getNextScannedResult()
+ throws IOException, FilterUnsupportedException {
+ if (dataBlockIterator.hasNext()) {
+ blocksChunkHolder.setDataBlock(dataBlockIterator.next());
+ blocksChunkHolder.reset();
+ return blockletScanner.scanBlocklet(blocksChunkHolder);
+ }
+ return null;
+ }
+
+ public abstract void processNextBatch(CarbonColumnarBatch columnarBatch);
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/processor/BlockletIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlockletIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/BlockletIterator.java
new file mode 100644
index 0000000..f06aa6a
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/BlockletIterator.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.processor;
+
+import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.core.datastore.DataRefNode;
+
+/**
+ * Below class will be used to iterate over data block
+ */
+public class BlockletIterator extends CarbonIterator<DataRefNode> {
+ /**
+ * data store block
+ */
+ protected DataRefNode datablock;
+ /**
+ * block counter to keep a track how many block has been processed
+ */
+ private int blockCounter;
+
+ /**
+ * flag to be used to check any more data block is present or not
+ */
+ private boolean hasNext = true;
+
+ /**
+ * total number blocks assgned to this iterator
+ */
+ private long totalNumberOfBlocksToScan;
+
+ /**
+ * Constructor
+ *
+ * @param datablock first data block
+ * @param totalNumberOfBlocksToScan total number of blocks to be scanned
+ */
+ public BlockletIterator(DataRefNode datablock, long totalNumberOfBlocksToScan) {
+ this.datablock = datablock;
+ this.totalNumberOfBlocksToScan = totalNumberOfBlocksToScan;
+ }
+
+ /**
+ * is all the blocks assigned to this iterator has been processed
+ */
+ @Override public boolean hasNext() {
+ return hasNext;
+ }
+
+ @Override
+ /**
+ * To get the next block
+ * @return next data block
+ *
+ */
+ public DataRefNode next() {
+ // get the current blocks
+ DataRefNode datablockTemp = datablock;
+ // store the next data block
+ datablock = datablock.getNextDataRefNode();
+ // increment the counter
+ blockCounter++;
+ // if all the data block is processed then
+ // set the has next flag to false
+ // or if number of blocks assigned to this iterator is processed
+ // then also set the hasnext flag to false
+ if (null == datablock || blockCounter >= this.totalNumberOfBlocksToScan) {
+ hasNext = false;
+ }
+ return datablockTemp;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
new file mode 100644
index 0000000..81628b3
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.processor;
+
+import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
+import org.apache.carbondata.core.datastorage.FileHolder;
+
+/**
+ * Block chunk holder which will hold the dimension and
+ * measure chunk
+ */
+public class BlocksChunkHolder {
+
+ /**
+ * dimension column data chunk
+ */
+ private DimensionColumnDataChunk[] dimensionDataChunk;
+
+ /**
+ * measure column data chunk
+ */
+ private MeasureColumnDataChunk[] measureDataChunk;
+
+ /**
+ * file reader which will use to read the block from file
+ */
+ private FileHolder fileReader;
+
+ /**
+ * data block
+ */
+ private DataRefNode dataBlock;
+
+ public BlocksChunkHolder(int numberOfDimensionBlock, int numberOfMeasureBlock) {
+ dimensionDataChunk = new DimensionColumnDataChunk[numberOfDimensionBlock];
+ measureDataChunk = new MeasureColumnDataChunk[numberOfMeasureBlock];
+ }
+
+ /**
+ * @return the dimensionDataChunk
+ */
+ public DimensionColumnDataChunk[] getDimensionDataChunk() {
+ return dimensionDataChunk;
+ }
+
+ /**
+ * @param dimensionDataChunk the dimensionDataChunk to set
+ */
+ public void setDimensionDataChunk(DimensionColumnDataChunk[] dimensionDataChunk) {
+ this.dimensionDataChunk = dimensionDataChunk;
+ }
+
+ /**
+ * @return the measureDataChunk
+ */
+ public MeasureColumnDataChunk[] getMeasureDataChunk() {
+ return measureDataChunk;
+ }
+
+ /**
+ * @param measureDataChunk the measureDataChunk to set
+ */
+ public void setMeasureDataChunk(MeasureColumnDataChunk[] measureDataChunk) {
+ this.measureDataChunk = measureDataChunk;
+ }
+
+ /**
+ * @return the fileReader
+ */
+ public FileHolder getFileReader() {
+ return fileReader;
+ }
+
+ /**
+ * @param fileReader the fileReader to set
+ */
+ public void setFileReader(FileHolder fileReader) {
+ this.fileReader = fileReader;
+ }
+
+ /**
+ * @return the dataBlock
+ */
+ public DataRefNode getDataBlock() {
+ return dataBlock;
+ }
+
+ /**
+ * @param dataBlock the dataBlock to set
+ */
+ public void setDataBlock(DataRefNode dataBlock) {
+ this.dataBlock = dataBlock;
+ }
+
+ /***
+ * To reset the measure chunk and dimension chunk
+ * array
+ */
+ public void reset() {
+ for (int i = 0; i < measureDataChunk.length; i++) {
+ this.measureDataChunk[i] = null;
+ }
+ for (int i = 0; i < dimensionDataChunk.length; i++) {
+ this.dimensionDataChunk[i] = null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java
new file mode 100644
index 0000000..4de9685
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.processor.impl;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.core.stats.QueryStatisticsModel;
+import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.processor.AbstractDataBlockIterator;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
+
+/**
+ * Below class will be used to process the block for detail query
+ */
+public class DataBlockIteratorImpl extends AbstractDataBlockIterator {
+ /**
+ * DataBlockIteratorImpl Constructor
+ *
+ * @param blockExecutionInfo execution information
+ */
+ public DataBlockIteratorImpl(BlockExecutionInfo blockExecutionInfo, FileHolder fileReader,
+ int batchSize, QueryStatisticsModel queryStatisticsModel) {
+ super(blockExecutionInfo, fileReader, batchSize, queryStatisticsModel);
+ }
+
+ /**
+ * It scans the block and returns the result with @batchSize
+ *
+ * @return Result of @batchSize
+ */
+ public List<Object[]> next() {
+ List<Object[]> collectedResult = null;
+ if (updateScanner()) {
+ collectedResult = this.scannerResultAggregator.collectData(scannedResult, batchSize);
+ while (collectedResult.size() < batchSize && updateScanner()) {
+ List<Object[]> data = this.scannerResultAggregator
+ .collectData(scannedResult, batchSize - collectedResult.size());
+ collectedResult.addAll(data);
+ }
+ } else {
+ collectedResult = new ArrayList<>();
+ }
+ return collectedResult;
+ }
+
+ public void processNextBatch(CarbonColumnarBatch columnarBatch) {
+ if (updateScanner()) {
+ this.scannerResultAggregator.collectVectorBatch(scannedResult, columnarBatch);
+ while (columnarBatch.getActualSize() < columnarBatch.getBatchSize() && updateScanner()) {
+ this.scannerResultAggregator.collectVectorBatch(scannedResult, columnarBatch);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
new file mode 100644
index 0000000..82649b4
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
@@ -0,0 +1,570 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.result;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.Map;
+
+import org.apache.carbondata.core.update.data.BlockletLevelDeleteDeltaDataCache;
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
+import org.apache.carbondata.core.path.CarbonTablePath;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.filter.GenericQueryType;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
+import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
+
+/**
+ * Scanned result class which will store and provide the result on request
+ */
+public abstract class AbstractScannedResult {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(AbstractScannedResult.class.getName());
+ /**
+ * current row number
+ */
+ protected int currentRow = -1;
+ /**
+ * row mapping indexes
+ */
+ protected int[] rowMapping;
+ /**
+ * key size of the fixed length column
+ */
+ private int fixedLengthKeySize;
+ /**
+ * total number of rows
+ */
+ private int totalNumberOfRows;
+ /**
+ * to keep track of number of rows process
+ */
+ protected int rowCounter;
+ /**
+ * dimension column data chunk
+ */
+ protected DimensionColumnDataChunk[] dataChunks;
+ /**
+ * measure column data chunk
+ */
+ protected MeasureColumnDataChunk[] measureDataChunks;
+ /**
+ * dictionary column block index in file
+ */
+ protected int[] dictionaryColumnBlockIndexes;
+
+ /**
+ * no dictionary column block index in file
+ */
+ protected int[] noDictionaryColumnBlockIndexes;
+
+ /**
+ * column group to is key structure info
+ * which will be used to get the key from the complete
+ * column group key
+ * For example if only one dimension of the column group is selected
+ * then from complete column group key it will be used to mask the key and
+ * get the particular column key
+ */
+ protected Map<Integer, KeyStructureInfo> columnGroupKeyStructureInfo;
+
+ /**
+ *
+ */
+ private Map<Integer, GenericQueryType> complexParentIndexToQueryMap;
+
+ private int totalDimensionsSize;
+
+ /**
+ * blockedId which will be blockId + blocklet number in the block
+ */
+ private String blockletId;
+
+ private long rowId;
+
+ /**
+ * parent block indexes
+ */
+ private int[] complexParentBlockIndexes;
+
+ protected BlockletLevelDeleteDeltaDataCache blockletDeleteDeltaCache;
+
+ public AbstractScannedResult(BlockExecutionInfo blockExecutionInfo) {
+ this.fixedLengthKeySize = blockExecutionInfo.getFixedLengthKeySize();
+ this.noDictionaryColumnBlockIndexes = blockExecutionInfo.getNoDictionaryBlockIndexes();
+ this.dictionaryColumnBlockIndexes = blockExecutionInfo.getDictionaryColumnBlockIndex();
+ this.columnGroupKeyStructureInfo = blockExecutionInfo.getColumnGroupToKeyStructureInfo();
+ this.complexParentIndexToQueryMap = blockExecutionInfo.getComlexDimensionInfoMap();
+ this.complexParentBlockIndexes = blockExecutionInfo.getComplexColumnParentBlockIndexes();
+ this.totalDimensionsSize = blockExecutionInfo.getQueryDimensions().length;
+ }
+
+ /**
+ * Below method will be used to set the dimension chunks
+ * which will be used to create a row
+ *
+ * @param dataChunks dimension chunks used in query
+ */
+ public void setDimensionChunks(DimensionColumnDataChunk[] dataChunks) {
+ this.dataChunks = dataChunks;
+ }
+
+ /**
+ * Below method will be used to set the measure column chunks
+ *
+ * @param measureDataChunks measure data chunks
+ */
+ public void setMeasureChunks(MeasureColumnDataChunk[] measureDataChunks) {
+ this.measureDataChunks = measureDataChunks;
+ }
+
+ /**
+ * Below method will be used to get the chunk based in measure ordinal
+ *
+ * @param ordinal measure ordinal
+ * @return measure column chunk
+ */
+ public MeasureColumnDataChunk getMeasureChunk(int ordinal) {
+ return measureDataChunks[ordinal];
+ }
+
+ /**
+ * Below method will be used to get the key for all the dictionary dimensions
+ * which is present in the query
+ *
+ * @param rowId row id selected after scanning
+ * @return return the dictionary key
+ */
+ protected byte[] getDictionaryKeyArray(int rowId) {
+ byte[] completeKey = new byte[fixedLengthKeySize];
+ int offset = 0;
+ for (int i = 0; i < this.dictionaryColumnBlockIndexes.length; i++) {
+ offset += dataChunks[dictionaryColumnBlockIndexes[i]]
+ .fillChunkData(completeKey, offset, rowId,
+ columnGroupKeyStructureInfo.get(dictionaryColumnBlockIndexes[i]));
+ }
+ rowCounter++;
+ return completeKey;
+ }
+
+ /**
+ * Below method will be used to get the key for all the dictionary dimensions
+ * in integer array format which is present in the query
+ *
+ * @param rowId row id selected after scanning
+ * @return return the dictionary key
+ */
+ protected int[] getDictionaryKeyIntegerArray(int rowId) {
+ int[] completeKey = new int[totalDimensionsSize];
+ int column = 0;
+ for (int i = 0; i < this.dictionaryColumnBlockIndexes.length; i++) {
+ column = dataChunks[dictionaryColumnBlockIndexes[i]]
+ .fillConvertedChunkData(rowId, column, completeKey,
+ columnGroupKeyStructureInfo.get(dictionaryColumnBlockIndexes[i]));
+ }
+ rowCounter++;
+ return completeKey;
+ }
+
+ /**
+ * Fill the column data of dictionary to vector
+ */
+ public void fillColumnarDictionaryBatch(ColumnVectorInfo[] vectorInfo) {
+ int column = 0;
+ for (int i = 0; i < this.dictionaryColumnBlockIndexes.length; i++) {
+ column = dataChunks[dictionaryColumnBlockIndexes[i]]
+ .fillConvertedChunkData(vectorInfo, column,
+ columnGroupKeyStructureInfo.get(dictionaryColumnBlockIndexes[i]));
+ }
+ }
+
+ /**
+ * Fill the column data to vector
+ */
+ public void fillColumnarNoDictionaryBatch(ColumnVectorInfo[] vectorInfo) {
+ int column = 0;
+ for (int i = 0; i < this.noDictionaryColumnBlockIndexes.length; i++) {
+ column = dataChunks[noDictionaryColumnBlockIndexes[i]]
+ .fillConvertedChunkData(vectorInfo, column,
+ columnGroupKeyStructureInfo.get(noDictionaryColumnBlockIndexes[i]));
+ }
+ }
+
+ /**
+ * Fill the measure column data to vector
+ */
+ public void fillColumnarMeasureBatch(ColumnVectorInfo[] vectorInfo, int[] measuresOrdinal) {
+ for (int i = 0; i < measuresOrdinal.length; i++) {
+ vectorInfo[i].measureVectorFiller
+ .fillMeasureVector(measureDataChunks[measuresOrdinal[i]], vectorInfo[i]);
+ }
+ }
+
+ public void fillColumnarComplexBatch(ColumnVectorInfo[] vectorInfos) {
+ for (int i = 0; i < vectorInfos.length; i++) {
+ int offset = vectorInfos[i].offset;
+ int len = offset + vectorInfos[i].size;
+ int vectorOffset = vectorInfos[i].vectorOffset;
+ CarbonColumnVector vector = vectorInfos[i].vector;
+ for (int j = offset; j < len; j++) {
+ ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+ DataOutputStream dataOutput = new DataOutputStream(byteStream);
+ try {
+ vectorInfos[i].genericQueryType.parseBlocksAndReturnComplexColumnByteArray(dataChunks,
+ rowMapping == null ? j : rowMapping[j], dataOutput);
+ Object data = vectorInfos[i].genericQueryType
+ .getDataBasedOnDataTypeFromSurrogates(ByteBuffer.wrap(byteStream.toByteArray()));
+ vector.putObject(vectorOffset++, data);
+ } catch (IOException e) {
+ LOGGER.error(e);
+ } finally {
+ CarbonUtil.closeStreams(dataOutput);
+ CarbonUtil.closeStreams(byteStream);
+ }
+ }
+ }
+ }
+
+ /**
+ * Just increment the counter incase of query only on measures.
+ */
+ public void incrementCounter() {
+ rowCounter++;
+ currentRow++;
+ }
+
+ /**
+ * increment the counter.
+ */
+ public void setRowCounter(int rowCounter) {
+ this.rowCounter = rowCounter;
+ }
+
+ /**
+ * Below method will be used to get the dimension data based on dimension
+ * ordinal and index
+ *
+ * @param dimOrdinal dimension ordinal present in the query
+ * @param rowId row index
+ * @return dimension data based on row id
+ */
+ protected byte[] getDimensionData(int dimOrdinal, int rowId) {
+ return dataChunks[dimOrdinal].getChunkData(rowId);
+ }
+
+ /**
+ * Below method will be used to get the dimension key array
+ * for all the no dictionary dimension present in the query
+ *
+ * @param rowId row number
+ * @return no dictionary keys for all no dictionary dimension
+ */
+ protected byte[][] getNoDictionaryKeyArray(int rowId) {
+ byte[][] noDictionaryColumnsKeys = new byte[noDictionaryColumnBlockIndexes.length][];
+ int position = 0;
+ for (int i = 0; i < this.noDictionaryColumnBlockIndexes.length; i++) {
+ noDictionaryColumnsKeys[position++] =
+ dataChunks[noDictionaryColumnBlockIndexes[i]].getChunkData(rowId);
+ }
+ return noDictionaryColumnsKeys;
+ }
+
+ /**
+ * Below method will be used to get the dimension key array
+ * for all the no dictionary dimension present in the query
+ *
+ * @param rowId row number
+ * @return no dictionary keys for all no dictionary dimension
+ */
+ protected String[] getNoDictionaryKeyStringArray(int rowId) {
+ String[] noDictionaryColumnsKeys = new String[noDictionaryColumnBlockIndexes.length];
+ int position = 0;
+ for (int i = 0; i < this.noDictionaryColumnBlockIndexes.length; i++) {
+ noDictionaryColumnsKeys[position++] =
+ new String(dataChunks[noDictionaryColumnBlockIndexes[i]].getChunkData(rowId));
+ }
+ return noDictionaryColumnsKeys;
+ }
+
+ /**
+ * @return blockletId
+ */
+ public String getBlockletId() {
+ return blockletId;
+ }
+
+ /**
+ * @param blockletId
+ */
+ public void setBlockletId(String blockletId) {
+ this.blockletId = CarbonTablePath.getShortBlockId(blockletId);
+ }
+
+ /**
+ * @return blockletId
+ */
+ public long getRowId() {
+ return rowId;
+ }
+
+ /**
+ * @param blockletId
+ */
+ public void setRowId(long rowId) {
+ this.rowId = rowId;
+ }
+
+
+ /**
+ * Below method will be used to get the complex type keys array based
+ * on row id for all the complex type dimension selected in query
+ *
+ * @param rowId row number
+ * @return complex type key array for all the complex dimension selected in query
+ */
+ protected byte[][] getComplexTypeKeyArray(int rowId) {
+ byte[][] complexTypeData = new byte[complexParentBlockIndexes.length][];
+ for (int i = 0; i < complexTypeData.length; i++) {
+ GenericQueryType genericQueryType =
+ complexParentIndexToQueryMap.get(complexParentBlockIndexes[i]);
+ ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+ DataOutputStream dataOutput = new DataOutputStream(byteStream);
+ try {
+ genericQueryType.parseBlocksAndReturnComplexColumnByteArray(dataChunks, rowId, dataOutput);
+ complexTypeData[i] = byteStream.toByteArray();
+ } catch (IOException e) {
+ LOGGER.error(e);
+ } finally {
+ CarbonUtil.closeStreams(dataOutput);
+ CarbonUtil.closeStreams(byteStream);
+ }
+ }
+ return complexTypeData;
+ }
+
+ /**
+ * @return return the total number of row after scanning
+ */
+ public int numberOfOutputRows() {
+ return this.totalNumberOfRows;
+ }
+
+ /**
+ * to check whether any more row is present in the result
+ *
+ * @return
+ */
+ public boolean hasNext() {
+ if (rowCounter < this.totalNumberOfRows) {
+ return true;
+ }
+ CarbonUtil.freeMemory(dataChunks, measureDataChunks);
+ return false;
+ }
+
+ /**
+ * As this class will be a flyweight object so
+ * for one block all the blocklet scanning will use same result object
+ * in that case we need to reset the counter to zero so
+ * for new result it will give the result from zero
+ */
+ public void reset() {
+ rowCounter = 0;
+ currentRow = -1;
+ }
+
+ /**
+ * @param totalNumberOfRows set total of number rows valid after scanning
+ */
+ public void setNumberOfRows(int totalNumberOfRows) {
+ this.totalNumberOfRows = totalNumberOfRows;
+ }
+
+ /**
+ * After applying filter it will return the bit set with the valid row indexes
+ * so below method will be used to set the row indexes
+ *
+ * @param indexes
+ */
+ public void setIndexes(int[] indexes) {
+ this.rowMapping = indexes;
+ }
+
+ /**
+ * Below method will be used to check whether measure value is null or not
+ *
+ * @param ordinal measure ordinal
+ * @param rowIndex row number to be checked
+ * @return whether it is null or not
+ */
+ protected boolean isNullMeasureValue(int ordinal, int rowIndex) {
+ return measureDataChunks[ordinal].getNullValueIndexHolder().getBitSet().get(rowIndex);
+ }
+
+ /**
+ * Below method will be used to get the measure value of
+ * long type
+ *
+ * @param ordinal measure ordinal
+ * @param rowIndex row number of the measure value
+ * @return measure value of long type
+ */
+ protected long getLongMeasureValue(int ordinal, int rowIndex) {
+ return measureDataChunks[ordinal].getMeasureDataHolder().getReadableLongValueByIndex(rowIndex);
+ }
+
+ /**
+ * Below method will be used to get the measure value of double type
+ *
+ * @param ordinal measure ordinal
+ * @param rowIndex row number
+ * @return measure value of double type
+ */
+ protected double getDoubleMeasureValue(int ordinal, int rowIndex) {
+ return measureDataChunks[ordinal].getMeasureDataHolder()
+ .getReadableDoubleValueByIndex(rowIndex);
+ }
+
+ /**
+ * Below method will be used to get the measure type of big decimal data type
+ *
+ * @param ordinal ordinal of the of the measure
+ * @param rowIndex row number
+ * @return measure of big decimal type
+ */
+ protected BigDecimal getBigDecimalMeasureValue(int ordinal, int rowIndex) {
+ return measureDataChunks[ordinal].getMeasureDataHolder()
+ .getReadableBigDecimalValueByIndex(rowIndex);
+ }
+
+ public int getRowCounter() {
+ return rowCounter;
+ }
+
+ /**
+ * will return the current valid row id
+ *
+ * @return valid row id
+ */
+ public abstract int getCurrenrRowId();
+
+ /**
+ * @return dictionary key array for all the dictionary dimension
+ * selected in query
+ */
+ public abstract byte[] getDictionaryKeyArray();
+
+ /**
+ * @return dictionary key array for all the dictionary dimension in integer array forat
+ * selected in query
+ */
+ public abstract int[] getDictionaryKeyIntegerArray();
+
+ /**
+ * Return the dimension data based on dimension ordinal
+ *
+ * @param dimensionOrdinal dimension ordinal
+ * @return dimension data
+ */
+ public abstract byte[] getDimensionKey(int dimensionOrdinal);
+
+ /**
+ * Below method will be used to get the complex type key array
+ *
+ * @return complex type key array
+ */
+ public abstract byte[][] getComplexTypeKeyArray();
+
+ /**
+ * Below method will be used to get the no dictionary key
+ * array for all the no dictionary dimension selected in query
+ *
+ * @return no dictionary key array for all the no dictionary dimension
+ */
+ public abstract byte[][] getNoDictionaryKeyArray();
+
+ /**
+ * Below method will be used to get the no dictionary key
+ * array in string array format for all the no dictionary dimension selected in query
+ *
+ * @return no dictionary key array for all the no dictionary dimension
+ */
+ public abstract String[] getNoDictionaryKeyStringArray();
+
+ /**
+ * Below method will be used to to check whether measure value
+ * is null or for a measure
+ *
+ * @param ordinal measure ordinal
+ * @return is null or not
+ */
+ public abstract boolean isNullMeasureValue(int ordinal);
+
+ /**
+ * Below method will be used to get the measure value for measure
+ * of long data type
+ *
+ * @param ordinal measure ordinal
+ * @return long value of measure
+ */
+ public abstract long getLongMeasureValue(int ordinal);
+
+ /**
+ * Below method will be used to get the value of measure of double
+ * type
+ *
+ * @param ordinal measure ordinal
+ * @return measure value
+ */
+ public abstract double getDoubleMeasureValue(int ordinal);
+
+ /**
+ * Below method will be used to get the data of big decimal type
+ * of a measure
+ *
+ * @param ordinal measure ordinal
+ * @return measure value
+ */
+ public abstract BigDecimal getBigDecimalMeasureValue(int ordinal);
+
+ /**
+ *
+ * @return BlockletLevelDeleteDeltaDataCache.
+ */
+ public BlockletLevelDeleteDeltaDataCache getDeleteDeltaDataCache() {
+ return blockletDeleteDeltaCache;
+ }
+
+ /**
+ * @param blockletDeleteDeltaCache
+ */
+ public void setBlockletDeleteDeltaCache(
+ BlockletLevelDeleteDeltaDataCache blockletDeleteDeltaCache) {
+ this.blockletDeleteDeltaCache = blockletDeleteDeltaCache;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/result/BatchResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/BatchResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/BatchResult.java
new file mode 100644
index 0000000..1c805ae
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/BatchResult.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.result;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import org.apache.carbondata.common.CarbonIterator;
+
+/**
+ * Below class holds the query result
+ */
+public class BatchResult extends CarbonIterator<Object[]> {
+
+ /**
+ * list of keys
+ */
+ protected List<Object[]> rows;
+
+ /**
+ * counter to check whether all the records are processed or not
+ */
+ protected int counter;
+
+ public BatchResult() {
+ this.rows = new ArrayList<>();
+ }
+
+ /**
+ * Below method will be used to get the rows
+ *
+ * @return
+ */
+ public List<Object[]> getRows() {
+ return rows;
+ }
+
+ /**
+ * Below method will be used to get the set the values
+ *
+ * @param rows
+ */
+ public void setRows(List<Object[]> rows) {
+ this.rows = rows;
+ }
+
+ /**
+ * This method will return one row at a time based on the counter given.
+ * @param counter
+ * @return
+ */
+ public Object[] getRawRow(int counter) {
+ return rows.get(counter);
+ }
+
+ /**
+ * For getting the total size.
+ * @return
+ */
+ public int getSize() {
+ return rows.size();
+ }
+
+
+ /**
+ * Returns {@code true} if the iteration has more elements.
+ *
+ * @return {@code true} if the iteration has more elements
+ */
+ @Override public boolean hasNext() {
+ return counter < rows.size();
+ }
+
+ /**
+ * Returns the next element in the iteration.
+ *
+ * @return the next element in the iteration
+ */
+ @Override public Object[] next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ Object[] row = rows.get(counter);
+ counter++;
+ return row;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java
new file mode 100644
index 0000000..7fc964b
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/impl/FilterQueryScannedResult.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.result.impl;
+
+import java.math.BigDecimal;
+
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
+
+/**
+ * Result provider class in case of filter query
+ * In case of filter query data will be send
+ * based on filtered row index
+ */
+public class FilterQueryScannedResult extends AbstractScannedResult {
+
+ public FilterQueryScannedResult(BlockExecutionInfo tableBlockExecutionInfos) {
+ super(tableBlockExecutionInfos);
+ }
+
+ /**
+ * @return dictionary key array for all the dictionary dimension
+ * selected in query
+ */
+ @Override public byte[] getDictionaryKeyArray() {
+ ++currentRow;
+ return getDictionaryKeyArray(rowMapping[currentRow]);
+ }
+
+ /**
+ * @return dictionary key integer array for all the dictionary dimension
+ * selected in query
+ */
+ @Override public int[] getDictionaryKeyIntegerArray() {
+ ++currentRow;
+ return getDictionaryKeyIntegerArray(rowMapping[currentRow]);
+ }
+
+ /**
+ * Below method will be used to get the complex type key array
+ *
+ * @return complex type key array
+ */
+ @Override public byte[][] getComplexTypeKeyArray() {
+ return getComplexTypeKeyArray(rowMapping[currentRow]);
+ }
+
+ /**
+ * Below method will be used to get the no dictionary key
+ * array for all the no dictionary dimension selected in query
+ *
+ * @return no dictionary key array for all the no dictionary dimension
+ */
+ @Override public byte[][] getNoDictionaryKeyArray() {
+ return getNoDictionaryKeyArray(rowMapping[currentRow]);
+ }
+
+ /**
+ * Below method will be used to get the no dictionary key
+ * string array for all the no dictionary dimension selected in query
+ *
+ * @return no dictionary key array for all the no dictionary dimension
+ */
+ @Override public String[] getNoDictionaryKeyStringArray() {
+ return getNoDictionaryKeyStringArray(rowMapping[currentRow]);
+ }
+
+ /**
+ * will return the current valid row id
+ *
+ * @return valid row id
+ */
+ @Override public int getCurrenrRowId() {
+ return rowMapping[currentRow];
+ }
+
+ /**
+ * Return the dimension data based on dimension ordinal
+ *
+ * @param dimensionOrdinal dimension ordinal
+ * @return dimension data
+ */
+ @Override public byte[] getDimensionKey(int dimensionOrdinal) {
+ return getDimensionData(dimensionOrdinal, rowMapping[currentRow]);
+ }
+
+ /**
+ * Below method will be used to to check whether measure value
+ * is null or for a measure
+ *
+ * @param ordinal measure ordinal
+ * @return is null or not
+ */
+ @Override public boolean isNullMeasureValue(int ordinal) {
+ return isNullMeasureValue(ordinal, rowMapping[currentRow]);
+ }
+
+ /**
+ * Below method will be used to get the measure value for measure
+ * of long data type
+ *
+ * @param ordinal measure ordinal
+ * @return long value of measure
+ */
+ @Override public long getLongMeasureValue(int ordinal) {
+ return getLongMeasureValue(ordinal, rowMapping[currentRow]);
+ }
+
+ /**
+ * Below method will be used to get the value of measure of double
+ * type
+ *
+ * @param ordinal measure ordinal
+ * @return measure value
+ */
+ @Override public double getDoubleMeasureValue(int ordinal) {
+ return getDoubleMeasureValue(ordinal, rowMapping[currentRow]);
+ }
+
+ /**
+ * Below method will be used to get the data of big decimal type
+ * of a measure
+ *
+ * @param ordinal measure ordinal
+ * @return measure value
+ */
+ @Override public BigDecimal getBigDecimalMeasureValue(int ordinal) {
+ return getBigDecimalMeasureValue(ordinal, rowMapping[currentRow]);
+ }
+
+ /**
+ * Fill the column data to vector
+ */
+ public void fillColumnarDictionaryBatch(ColumnVectorInfo[] vectorInfo) {
+ int column = 0;
+ for (int i = 0; i < this.dictionaryColumnBlockIndexes.length; i++) {
+ column = dataChunks[dictionaryColumnBlockIndexes[i]]
+ .fillConvertedChunkData(rowMapping, vectorInfo, column,
+ columnGroupKeyStructureInfo.get(dictionaryColumnBlockIndexes[i]));
+ }
+ }
+
+ /**
+ * Fill the column data to vector
+ */
+ public void fillColumnarNoDictionaryBatch(ColumnVectorInfo[] vectorInfo) {
+ int column = 0;
+ for (int i = 0; i < this.noDictionaryColumnBlockIndexes.length; i++) {
+ column = dataChunks[noDictionaryColumnBlockIndexes[i]]
+ .fillConvertedChunkData(rowMapping, vectorInfo, column,
+ columnGroupKeyStructureInfo.get(noDictionaryColumnBlockIndexes[i]));
+ }
+ }
+
+ /**
+ * Fill the measure column data to vector
+ */
+ public void fillColumnarMeasureBatch(ColumnVectorInfo[] vectorInfo, int[] measuresOrdinal) {
+ for (int i = 0; i < measuresOrdinal.length; i++) {
+ vectorInfo[i].measureVectorFiller
+ .fillMeasureVectorForFilter(rowMapping, measureDataChunks[measuresOrdinal[i]],
+ vectorInfo[i]);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java
new file mode 100644
index 0000000..5cfea25
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/impl/NonFilterQueryScannedResult.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.result.impl;
+
+import java.math.BigDecimal;
+
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+
+/**
+ * Result provide class for non filter query
+ * In case of no filter query we need to return
+ * complete data
+ */
+public class NonFilterQueryScannedResult extends AbstractScannedResult {
+
+ public NonFilterQueryScannedResult(BlockExecutionInfo blockExecutionInfo) {
+ super(blockExecutionInfo);
+ }
+
+ /**
+ * @return dictionary key array for all the dictionary dimension selected in
+ * query
+ */
+ @Override public byte[] getDictionaryKeyArray() {
+ ++currentRow;
+ return getDictionaryKeyArray(currentRow);
+ }
+
+ /**
+ * @return dictionary key integer array for all the dictionary dimension
+ * selected in query
+ */
+ @Override public int[] getDictionaryKeyIntegerArray() {
+ ++currentRow;
+ return getDictionaryKeyIntegerArray(currentRow);
+ }
+
+ /**
+ * Below method will be used to get the complex type key array
+ *
+ * @return complex type key array
+ */
+ @Override public byte[][] getComplexTypeKeyArray() {
+ return getComplexTypeKeyArray(currentRow);
+ }
+
+ /**
+ * Below method will be used to get the no dictionary key array for all the
+ * no dictionary dimension selected in query
+ *
+ * @return no dictionary key array for all the no dictionary dimension
+ */
+ @Override public byte[][] getNoDictionaryKeyArray() {
+ return getNoDictionaryKeyArray(currentRow);
+ }
+
+ /**
+ * Below method will be used to get the no dictionary key
+ * string array for all the no dictionary dimension selected in query
+ *
+ * @return no dictionary key array for all the no dictionary dimension
+ */
+ @Override public String[] getNoDictionaryKeyStringArray() {
+ return getNoDictionaryKeyStringArray(currentRow);
+ }
+
+ /**
+ * will return the current valid row id
+ *
+ * @return valid row id
+ */
+ @Override public int getCurrenrRowId() {
+ return currentRow;
+ }
+
+ /**
+ * Return the dimension data based on dimension ordinal
+ *
+ * @param dimensionOrdinal dimension ordinal
+ * @return dimension data
+ */
+ @Override public byte[] getDimensionKey(int dimensionOrdinal) {
+ return getDimensionData(dimensionOrdinal, currentRow);
+ }
+
+ /**
+ * Below method will be used to to check whether measure value is null or
+ * for a measure
+ *
+ * @param ordinal measure ordinal
+ * @return is null or not
+ */
+ @Override public boolean isNullMeasureValue(int ordinal) {
+ return isNullMeasureValue(ordinal, currentRow);
+ }
+
+ /**
+ * Below method will be used to get the measure value for measure of long
+ * data type
+ *
+ * @param ordinal measure ordinal
+ * @return long value of measure
+ */
+ @Override public long getLongMeasureValue(int ordinal) {
+ return getLongMeasureValue(ordinal, currentRow);
+ }
+
+ /**
+ * Below method will be used to get the value of measure of double type
+ *
+ * @param ordinal measure ordinal
+ * @return measure value
+ */
+ @Override public double getDoubleMeasureValue(int ordinal) {
+ return getDoubleMeasureValue(ordinal, currentRow);
+ }
+
+ /**
+ * Below method will be used to get the data of big decimal type of a
+ * measure
+ *
+ * @param ordinal measure ordinal
+ * @return measure value
+ */
+ @Override public BigDecimal getBigDecimalMeasureValue(int ordinal) {
+ return getBigDecimalMeasureValue(ordinal, currentRow);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
new file mode 100644
index 0000000..88c0144
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.result.iterator;
+
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+
+import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.datastore.DataRefNodeFinder;
+import org.apache.carbondata.core.datastore.impl.btree.BTreeDataRefNodeFinder;
+import org.apache.carbondata.core.stats.QueryStatistic;
+import org.apache.carbondata.core.stats.QueryStatisticsConstants;
+import org.apache.carbondata.core.stats.QueryStatisticsModel;
+import org.apache.carbondata.core.stats.QueryStatisticsRecorder;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.core.scan.processor.AbstractDataBlockIterator;
+import org.apache.carbondata.core.scan.processor.impl.DataBlockIteratorImpl;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
+
+/**
+ * In case of detail query we cannot keep all the records in memory so for
+ * executing that query are returning a iterator over block and every time next
+ * call will come it will execute the block and return the result
+ */
+public abstract class AbstractDetailQueryResultIterator<E> extends CarbonIterator<E> {
+
+ /**
+ * LOGGER.
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(AbstractDetailQueryResultIterator.class.getName());
+
+ protected ExecutorService execService;
+ /**
+ * execution info of the block
+ */
+ protected List<BlockExecutionInfo> blockExecutionInfos;
+
+ /**
+ * file reader which will be used to execute the query
+ */
+ protected FileHolder fileReader;
+ protected AbstractDataBlockIterator dataBlockIterator;
+ protected boolean nextBatch = false;
+ /**
+ * total time scan the blocks
+ */
+ protected long totalScanTime;
+ /**
+ * is the statistic recorded
+ */
+ protected boolean isStatisticsRecorded;
+ /**
+ * QueryStatisticsRecorder
+ */
+ protected QueryStatisticsRecorder recorder;
+ /**
+ * number of cores which can be used
+ */
+ private int batchSize;
+ /**
+ * queryStatisticsModel to store query statistics object
+ */
+ QueryStatisticsModel queryStatisticsModel;
+
+ public AbstractDetailQueryResultIterator(List<BlockExecutionInfo> infos, QueryModel queryModel,
+ ExecutorService execService) {
+ String batchSizeString =
+ CarbonProperties.getInstance().getProperty(CarbonCommonConstants.DETAIL_QUERY_BATCH_SIZE);
+ if (null != batchSizeString) {
+ try {
+ batchSize = Integer.parseInt(batchSizeString);
+ } catch (NumberFormatException ne) {
+ LOGGER.error("Invalid inmemory records size. Using default value");
+ batchSize = CarbonCommonConstants.DETAIL_QUERY_BATCH_SIZE_DEFAULT;
+ }
+ } else {
+ batchSize = CarbonCommonConstants.DETAIL_QUERY_BATCH_SIZE_DEFAULT;
+ }
+ this.recorder = queryModel.getStatisticsRecorder();
+ this.blockExecutionInfos = infos;
+ this.fileReader = FileFactory.getFileHolder(
+ FileFactory.getFileType(queryModel.getAbsoluteTableIdentifier().getStorePath()));
+ this.execService = execService;
+ intialiseInfos();
+ initQueryStatiticsModel();
+ }
+
+ private void intialiseInfos() {
+ totalScanTime = System.currentTimeMillis();
+ for (BlockExecutionInfo blockInfo : blockExecutionInfos) {
+ DataRefNodeFinder finder = new BTreeDataRefNodeFinder(blockInfo.getEachColumnValueSize());
+ DataRefNode startDataBlock = finder
+ .findFirstDataBlock(blockInfo.getDataBlock().getDataRefNode(), blockInfo.getStartKey());
+ while (startDataBlock.nodeNumber() < blockInfo.getStartBlockletIndex()) {
+ startDataBlock = startDataBlock.getNextDataRefNode();
+ }
+
+ long numberOfBlockToScan = blockInfo.getNumberOfBlockletToScan();
+ //if number of block is less than 0 then take end block.
+ if (numberOfBlockToScan <= 0) {
+ DataRefNode endDataBlock = finder
+ .findLastDataBlock(blockInfo.getDataBlock().getDataRefNode(), blockInfo.getEndKey());
+ numberOfBlockToScan = endDataBlock.nodeNumber() - startDataBlock.nodeNumber() + 1;
+ }
+ blockInfo.setFirstDataBlock(startDataBlock);
+ blockInfo.setNumberOfBlockToScan(numberOfBlockToScan);
+ }
+ }
+
+ @Override public boolean hasNext() {
+ if ((dataBlockIterator != null && dataBlockIterator.hasNext()) || nextBatch) {
+ return true;
+ } else if (blockExecutionInfos.size() > 0) {
+ return true;
+ } else {
+ if (!isStatisticsRecorded) {
+ QueryStatistic statistic = new QueryStatistic();
+ statistic.addFixedTimeStatistic(QueryStatisticsConstants.SCAN_BLOCKS_TIME,
+ System.currentTimeMillis() - totalScanTime);
+ recorder.recordStatistics(statistic);
+ isStatisticsRecorded = true;
+ }
+ return false;
+ }
+ }
+
+ protected void updateDataBlockIterator() {
+ if (dataBlockIterator == null || !dataBlockIterator.hasNext()) {
+ dataBlockIterator = getDataBlockIterator();
+ while (dataBlockIterator != null && !dataBlockIterator.hasNext()) {
+ dataBlockIterator = getDataBlockIterator();
+ }
+ }
+ }
+
+ private DataBlockIteratorImpl getDataBlockIterator() {
+ if (blockExecutionInfos.size() > 0) {
+ BlockExecutionInfo executionInfo = blockExecutionInfos.get(0);
+ blockExecutionInfos.remove(executionInfo);
+ queryStatisticsModel.setRecorder(recorder);
+ return new DataBlockIteratorImpl(executionInfo, fileReader, batchSize, queryStatisticsModel);
+ }
+ return null;
+ }
+
+ protected void initQueryStatiticsModel() {
+ this.queryStatisticsModel = new QueryStatisticsModel();
+ QueryStatistic queryStatisticTotalBlocklet = new QueryStatistic();
+ queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .put(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM, queryStatisticTotalBlocklet);
+ QueryStatistic queryStatisticValidScanBlocklet = new QueryStatistic();
+ queryStatisticsModel.getStatisticsTypeAndObjMap()
+ .put(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM, queryStatisticValidScanBlocklet);
+ }
+
+ public void processNextBatch(CarbonColumnarBatch columnarBatch) {
+ throw new UnsupportedOperationException("Please use VectorDetailQueryResultIterator");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/ChunkRowIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/ChunkRowIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/ChunkRowIterator.java
new file mode 100644
index 0000000..61518b2
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/ChunkRowIterator.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.result.iterator;
+
+import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.core.scan.result.BatchResult;
+
+/**
+ * Iterator over row result
+ */
+public class ChunkRowIterator extends CarbonIterator<Object[]> {
+
+ /**
+ * iterator over chunk result
+ */
+ private CarbonIterator<BatchResult> iterator;
+
+ /**
+ * currect chunk
+ */
+ private BatchResult currentchunk;
+
+ public ChunkRowIterator(CarbonIterator<BatchResult> iterator) {
+ this.iterator = iterator;
+ if (iterator.hasNext()) {
+ currentchunk = iterator.next();
+ }
+ }
+
+ /**
+ * Returns {@code true} if the iteration has more elements. (In other words,
+ * returns {@code true} if {@link #next} would return an element rather than
+ * throwing an exception.)
+ *
+ * @return {@code true} if the iteration has more elements
+ */
+ @Override public boolean hasNext() {
+ if (null != currentchunk) {
+ if ((currentchunk.hasNext())) {
+ return true;
+ } else if (!currentchunk.hasNext()) {
+ while (iterator.hasNext()) {
+ currentchunk = iterator.next();
+ if (currentchunk != null && currentchunk.hasNext()) {
+ return true;
+ }
+ }
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Returns the next element in the iteration.
+ *
+ * @return the next element in the iteration
+ */
+ @Override public Object[] next() {
+ return currentchunk.next();
+ }
+
+}
[35/57] [abbrv] incubator-carbondata git commit: move identifier to
metadata package
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java b/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
deleted file mode 100644
index 5f7190a..0000000
--- a/processing/src/test/java/org/apache/carbondata/lcm/locks/LocalFileLockTest.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.lcm.locks;
-
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.locks.LocalFileLock;
-import org.apache.carbondata.locks.LockUsage;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.pentaho.di.core.util.Assert;
-
-/**
- * Test class to test the functionality of the local file locking.
- *
- * @author Administrator
- */
-public class LocalFileLockTest {
-
- /**
- * @throws java.lang.Exception
- */
- @Before public void setUp() throws Exception {
- }
-
- /**
- * @throws java.lang.Exception
- */
- @After public void tearDown() throws Exception {
- }
-
- @Test public void testingLocalFileLockingByAcquiring2Locks() {
-
- CarbonTableIdentifier carbonTableIdentifier = new CarbonTableIdentifier("databaseName", "tableName", "tableId");
- LocalFileLock localLock1 =
- new LocalFileLock(carbonTableIdentifier,
- LockUsage.METADATA_LOCK);
- Assert.assertTrue(localLock1.lock());
- LocalFileLock localLock2 =
- new LocalFileLock(carbonTableIdentifier,
- LockUsage.METADATA_LOCK);
- Assert.assertTrue(!localLock2.lock());
-
- Assert.assertTrue(localLock1.unlock());
- Assert.assertTrue(localLock2.lock());
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/processing/src/test/java/org/apache/carbondata/lcm/locks/ZooKeeperLockingTest.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/lcm/locks/ZooKeeperLockingTest.java b/processing/src/test/java/org/apache/carbondata/lcm/locks/ZooKeeperLockingTest.java
deleted file mode 100644
index 6c21543..0000000
--- a/processing/src/test/java/org/apache/carbondata/lcm/locks/ZooKeeperLockingTest.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.lcm.locks;
-
-import mockit.NonStrictExpectations;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.locks.LockUsage;
-import org.apache.carbondata.locks.ZooKeeperLocking;
-import org.apache.carbondata.locks.ZookeeperInit;
-import org.apache.zookeeper.server.ServerConfig;
-import org.apache.zookeeper.server.ZooKeeperServerMain;
-import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.ServerSocket;
-import java.util.Properties;
-
-/**
- * @author Administrator
- */
-public class ZooKeeperLockingTest {
-
- int freePort;
-
- /**
- * @throws java.lang.Exception
- */
- @Before public void setUp() throws Exception {
- Properties startupProperties = new Properties();
- startupProperties.setProperty("dataDir", (new File("./target").getAbsolutePath()));
- startupProperties.setProperty("dataLogDir", (new File("./target").getAbsolutePath()));
- freePort = findFreePort();
- startupProperties.setProperty("clientPort", "" + freePort);
- QuorumPeerConfig quorumConfiguration = new QuorumPeerConfig();
- try {
- quorumConfiguration.parseProperties(startupProperties);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
-
- final ZooKeeperServerMain zooKeeperServer = new ZooKeeperServerMain();
- final ServerConfig configuration = new ServerConfig();
- configuration.readFrom(quorumConfiguration);
- new Thread() {
- public void run() {
- try {
- zooKeeperServer.runFromConfig(configuration);
- } catch (IOException e) {
- System.out.println("ZooKeeper failure");
- }
- }
- }.start();
- }
-
- /**
- * @throws java.lang.Exception
- */
- @After public void tearDown() throws Exception {
- }
-
- @Test public void testZooKeeperLockingByTryingToAcquire2Locks()
- throws IllegalArgumentException, IllegalAccessException, NoSuchFieldException,
- SecurityException {
-
- final CarbonProperties cp = CarbonProperties.getInstance();
- new NonStrictExpectations(cp) {
- {
- cp.getProperty("/CarbonLocks");
- result = "/carbontests";
- cp.getProperty("spark.deploy.zookeeper.url");
- result = "127.0.0.1:" + freePort;
- }
- };
-
- ZookeeperInit zki = ZookeeperInit.getInstance("127.0.0.1:" + freePort);
-
- CarbonTableIdentifier tableIdentifier = new CarbonTableIdentifier("dbName", "tableName", "tableId");
- ZooKeeperLocking zkl =
- new ZooKeeperLocking(tableIdentifier,
- LockUsage.METADATA_LOCK);
- Assert.assertTrue(zkl.lock());
-
- ZooKeeperLocking zk2 = new ZooKeeperLocking(
- tableIdentifier, LockUsage.METADATA_LOCK);
- Assert.assertTrue(!zk2.lock());
-
- Assert.assertTrue(zkl.unlock());
- Assert.assertTrue(zk2.lock());
- Assert.assertTrue(zk2.unlock());
- }
-
- /**
- * For finding the free port available.
- *
- * @return
- */
- private static int findFreePort() {
- ServerSocket socket = null;
- try {
- socket = new ServerSocket(0);
- socket.setReuseAddress(true);
- int port = socket.getLocalPort();
- try {
- socket.close();
- } catch (IOException e) {
- // Ignore IOException on close()
- }
- return port;
- } catch (Exception e) {
- // Ignore
- } finally {
- if (socket != null) {
- try {
- socket.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
- return 2181;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java b/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
index 3afc0ae..a105eee 100644
--- a/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
+++ b/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
@@ -24,10 +24,10 @@ import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.cache.CacheType;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.CarbonDataLoadSchema;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.processing.model.CarbonDataLoadSchema;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.converter.SchemaConverter;
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl;
@@ -57,9 +57,9 @@ import org.apache.carbondata.core.writer.sortindex.CarbonDictionarySortIndexWrit
import org.apache.carbondata.core.writer.sortindex.CarbonDictionarySortIndexWriterImpl;
import org.apache.carbondata.core.writer.sortindex.CarbonDictionarySortInfo;
import org.apache.carbondata.core.writer.sortindex.CarbonDictionarySortInfoPreparator;
-import org.apache.carbondata.fileoperations.AtomicFileOperations;
-import org.apache.carbondata.fileoperations.AtomicFileOperationsImpl;
-import org.apache.carbondata.fileoperations.FileWriteOperation;
+import org.apache.carbondata.core.atomic.AtomicFileOperations;
+import org.apache.carbondata.core.atomic.AtomicFileOperationsImpl;
+import org.apache.carbondata.core.atomic.FileWriteOperation;
import org.apache.carbondata.processing.api.dataloader.DataLoadModel;
import org.apache.carbondata.processing.api.dataloader.SchemaInfo;
import org.apache.carbondata.processing.csvload.DataGraphExecuter;
[09/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/ViewFsCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/ViewFsCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/ViewFsCarbonFileTest.java
new file mode 100644
index 0000000..9cdef3a
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/ViewFsCarbonFileTest.java
@@ -0,0 +1,311 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.filesystem;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.viewfs.ViewFileSystem;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+
+public class ViewFsCarbonFileTest {
+
+ private static ViewFSCarbonFile viewFSCarbonFile;
+ private static FileStatus fileStatus;
+ private static FileStatus fileStatusWithOutDirectoryPermission;
+ private static String fileName;
+ private static File file;
+
+
+ @BeforeClass
+ static public void setUp() {
+ file = new File("Test.carbondata");
+ if (!file.exists())
+ try {
+ file.createNewFile();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
+ FileOutputStream oFile = new FileOutputStream(file, true);
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
+
+ fileStatus = new FileStatus(12L, true, 60, 120l, 180L, new Path(file.getAbsolutePath()));
+ fileStatusWithOutDirectoryPermission = new FileStatus(12L, false, 60, 120l, 180L, new Path(file.getAbsolutePath()));
+ fileName = file.getAbsolutePath();
+ viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
+ }
+
+ @AfterClass
+ static public void cleanUp() {
+ file.delete();
+ }
+
+ @Test
+ public void testRenameForceForException() throws IOException {
+
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ throw new IOException();
+ }
+
+ };
+ viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
+ viewFSCarbonFile.renameForce(fileName);
+ }
+
+ @Test
+ public void testListFilesWithOutDirectoryPermission() {
+ viewFSCarbonFile = new ViewFSCarbonFile(fileStatusWithOutDirectoryPermission);
+ assertTrue(viewFSCarbonFile.listFiles() == null);
+ }
+
+ @Test
+ public void testConstructorWithFilePath() {
+ viewFSCarbonFile = new ViewFSCarbonFile(file.getAbsolutePath());
+ assertTrue(viewFSCarbonFile instanceof ViewFSCarbonFile);
+ }
+
+ @Test
+ public void testListFilesForNullListStatus() {
+ viewFSCarbonFile = new ViewFSCarbonFile(fileStatusWithOutDirectoryPermission);
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new ViewFileSystem();
+ }
+
+ };
+ new MockUp<ViewFileSystem>() {
+ @Mock
+ public FileStatus[] listStatus(Path var1) throws IOException {
+
+ return null;
+ }
+
+ };
+ new MockUp<ViewFileSystem>() {
+ @Mock
+ public boolean delete(Path var1, boolean var2) throws IOException {
+
+ return true;
+ }
+
+ };
+ //public boolean delete(Path var1, boolean var2) throws IOException;
+ viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
+ assertTrue(viewFSCarbonFile.listFiles().length == 0);
+ }
+
+ @Test
+ public void testListDirectory() {
+ viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new ViewFileSystem();
+ }
+
+ };
+ new MockUp<ViewFileSystem>() {
+ @Mock
+ public FileStatus[] listStatus(Path var1) throws IOException {
+
+ FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
+ return fileStatus;
+ }
+
+ };
+
+ assertTrue(viewFSCarbonFile.listFiles().length == 1);
+ }
+
+ @Test
+ public void testListFilesForException() throws IOException {
+ viewFSCarbonFile = new ViewFSCarbonFile(fileStatusWithOutDirectoryPermission);
+
+ new MockUp<FileStatus>() {
+ @Mock
+ public Path getPath() {
+ return new Path(file.getAbsolutePath());
+ }
+
+ };
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ throw new IOException();
+ }
+
+ };
+ new MockUp<ViewFileSystem>() {
+ @Mock
+ public FileStatus[] listStatus(Path var1) throws IOException {
+
+ throw new IOException();
+ }
+
+ };
+ viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
+ viewFSCarbonFile.listFiles();
+ }
+
+ @Test
+ public void testListFilesWithCarbonFilter() {
+ CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
+
+ @Override
+ public boolean accept(CarbonFile file) {
+ return true;
+ }
+ };
+ viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
+ assertTrue(viewFSCarbonFile.listFiles(carbonFileFilter).length == 1);
+ }
+
+ @Test
+ public void testlistFilesWithoutFilter() {
+ CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
+
+ @Override
+ public boolean accept(CarbonFile file) {
+ return false;
+ }
+ };
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new ViewFileSystem();
+ }
+
+ };
+ new MockUp<ViewFileSystem>() {
+ @Mock
+ public FileStatus[] listStatus(Path var1) throws IOException {
+
+ FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
+ return fileStatus;
+ }
+
+ };
+ viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
+ assertTrue(viewFSCarbonFile.listFiles(carbonFileFilter).length == 0);
+ }
+
+ @Test
+ public void testGetParentFIle() {
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+ new MockUp<Path>() {
+ @Mock
+ public Path getParent() {
+ return new Path(file.getAbsolutePath()
+ );
+ }
+
+ };
+ new MockUp<FileStatus>() {
+ @Mock
+ public Path getPath() {
+ return new Path(file.getAbsolutePath());
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public FileStatus getFileStatus(Path f) throws IOException {
+
+ return new FileStatus(12L, true, 60, 120l, 180L, new Path(file.getAbsolutePath()));
+ }
+
+ };
+
+ viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
+ assertFalse(viewFSCarbonFile.getParentFile().equals(null));
+ }
+
+ @Test
+ public void testForNonDisributedSystem() {
+ viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new WebHdfsFileSystem();
+ }
+
+ };
+ assertFalse(viewFSCarbonFile.renameForce(fileName));
+ }
+
+ @Test
+ public void testrenameForceForViewFileSystem() {
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new ViewFileSystem();
+ }
+
+ };
+ new MockUp<ViewFileSystem>() {
+ @Mock
+ public boolean delete(Path f, boolean recursive) throws
+ IOException {
+ return true;
+
+ }
+
+ };
+ new MockUp<ViewFileSystem>() {
+ @Mock
+ public boolean rename(Path src, Path dst) throws IOException {
+ return true;
+
+ }
+
+ };
+
+ assertTrue(viewFSCarbonFile.renameForce(fileName));
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolderUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolderUnitTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolderUnitTest.java
deleted file mode 100644
index 9ff0677..0000000
--- a/core/src/test/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolderUnitTest.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.datastorage.store.columnar;
-
-import java.util.List;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-public class ColumnarKeyStoreDataHolderUnitTest {
-
- private static ColumnarKeyStoreDataHolder columnarKeyStoreDataHolder;
- private static ColumnarKeyStoreMetadata columnarKeyStoreMetadata;
-
- @BeforeClass public static void setup() {
- byte[] keyBlockData = new byte[] { 16, 8, 32, 40, 8, 8, 8 };
- int eachRowSize = 2;
- int[] reverseIndex = new int[] { 1, 5, 6, 3, 8 };
- columnarKeyStoreMetadata = new ColumnarKeyStoreMetadata(eachRowSize);
- columnarKeyStoreMetadata.setColumnReverseIndex(reverseIndex);
- columnarKeyStoreDataHolder =
- new ColumnarKeyStoreDataHolder(keyBlockData, columnarKeyStoreMetadata);
- }
-
- @Test public void testGetSurrogateKeyWithNullINGetColumnReverseIndex() {
- byte[] keyBlockData = new byte[] { 16, 8, 32, 40, 8, 8, 8 };
- int eachRowSize = 1;
- ColumnarKeyStoreMetadata columnarKeyStoreMetadata = new ColumnarKeyStoreMetadata(eachRowSize);
- ColumnarKeyStoreDataHolder columnarKeyStoreDataHolderNew =
- new ColumnarKeyStoreDataHolder(keyBlockData, columnarKeyStoreMetadata);
- int columnIndex = 5;
- int expected_result = 8;
- int result = columnarKeyStoreDataHolderNew.getSurrogateKey(columnIndex);
- assertEquals(expected_result, result);
- }
-
- @Test public void testGetSurrogateKeyWithNullINGetColumnReverseIndexAndRowSizeTwo() {
- byte[] keyBlockData = new byte[] { 16, 8, 32, 40, 8, 8, 8 };
- int eachRowSize = 2;
- ColumnarKeyStoreMetadata columnarKeyStoreMetadata = new ColumnarKeyStoreMetadata(eachRowSize);
- ColumnarKeyStoreDataHolder columnarKeyStoreDataHolderNew =
- new ColumnarKeyStoreDataHolder(keyBlockData, columnarKeyStoreMetadata);
- int columnIndex = 0;
- int expected_result = 4104;
- int result = columnarKeyStoreDataHolderNew.getSurrogateKey(columnIndex);
- assertEquals(expected_result, result);
- }
-
- @Test public void testGetSurrogateKeyWithNotNullINGetColumnReverseIndex() {
- int columnIndex = 0;
- int expected_result = 8232;
- int result = columnarKeyStoreDataHolder.getSurrogateKey(columnIndex);
- assertEquals(expected_result, result);
- }
-
- @Test(expected = ArrayIndexOutOfBoundsException.class)
- public void testExceptionInGetSurrogateKey() {
- int columnIndex = 10;
- int expected_result = 8232;
- int result = columnarKeyStoreDataHolder.getSurrogateKey(columnIndex);
- assertEquals(expected_result, result);
- }
-
- @Test public void testGetSurrogateKeyWithListOfByteWhileCreatingObject() {
- byte[] keyBlockData = new byte[] { 32, 64, 32, 40, 64, 8, 8 };
- List<byte[]> noDictionaryValBasedKeyBlockData = new java.util.ArrayList<>();
- noDictionaryValBasedKeyBlockData.add(keyBlockData);
- new ColumnarKeyStoreDataHolder(columnarKeyStoreMetadata);
- int columnIndex = 0;
- int expected_result = 8232;
- int result = columnarKeyStoreDataHolder.getSurrogateKey(columnIndex);
- assertEquals(expected_result, result);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/datastorage/store/filesystem/AlluxioCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/store/filesystem/AlluxioCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/store/filesystem/AlluxioCarbonFileTest.java
deleted file mode 100644
index f78e77c..0000000
--- a/core/src/test/java/org/apache/carbondata/core/datastorage/store/filesystem/AlluxioCarbonFileTest.java
+++ /dev/null
@@ -1,295 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.filesystem;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.apache.carbondata.core.datastorage.store.filesystem.AlluxioCarbonFile;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Options;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-public class AlluxioCarbonFileTest {
-
- private static AlluxioCarbonFile alluxioCarbonFile;
- private static FileStatus fileStatus;
- private static FileStatus fileStatusWithOutDirectoryPermission;
- private static String fileName;
- private static File file;
-
-
- @BeforeClass
- static public void setUp() {
- file = new File("Test.carbondata");
- if (!file.exists())
- try {
- file.createNewFile();
- } catch (IOException e) {
- e.printStackTrace();
- }
- try {
- FileOutputStream oFile = new FileOutputStream(file, true);
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- }
-
- fileStatus = new FileStatus(12L, true, 60, 120l, 180L, new Path(file.getAbsolutePath()));
- fileStatusWithOutDirectoryPermission = new FileStatus(12L, false, 60, 120l, 180L, new Path(file.getAbsolutePath()));
- fileName = file.getAbsolutePath();
- alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
- }
-
- @AfterClass
- static public void cleanUp() {
- file.delete();
- }
-
- @Test
- public void testRenameForceForException() throws IOException {
-
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- throw new IOException();
- }
-
- };
- alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
- alluxioCarbonFile.renameForce(fileName);
- }
-
- @Test
- public void testListFilesWithOutDirectoryPermission() {
- alluxioCarbonFile = new AlluxioCarbonFile(fileStatusWithOutDirectoryPermission);
- assertTrue(alluxioCarbonFile.listFiles() == null);
- }
-
- @Test
- public void testConstructorWithFilePath() {
- alluxioCarbonFile = new AlluxioCarbonFile(file.getAbsolutePath());
- assertTrue(alluxioCarbonFile instanceof AlluxioCarbonFile);
- }
-
- @Test
- public void testListFilesForNullListStatus() {
- alluxioCarbonFile = new AlluxioCarbonFile(fileStatusWithOutDirectoryPermission);
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public FileStatus[] listStatus(Path var1) throws IOException {
-
- return null;
- }
-
- };
- alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
- assertTrue(alluxioCarbonFile.listFiles().length == 0);
- }
-
- @Test
- public void testListDirectory() {
- alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public FileStatus[] listStatus(Path var1) throws IOException {
-
- FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
- }
-
- };
-
- assertTrue(alluxioCarbonFile.listFiles().length == 1);
- }
-
- @Test
- public void testListFilesForException() throws IOException {
- alluxioCarbonFile = new AlluxioCarbonFile(fileStatusWithOutDirectoryPermission);
-
- new MockUp<FileStatus>() {
- @Mock
- public Path getPath() {
- return new Path(file.getAbsolutePath());
- }
-
- };
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- throw new IOException();
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public FileStatus[] listStatus(Path var1) throws IOException {
-
- throw new IOException();
- }
-
- };
- alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
- alluxioCarbonFile.listFiles();
- }
-
- @Test
- public void testListFilesWithCarbonFilter() {
- CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
-
- @Override
- public boolean accept(CarbonFile file) {
- return true;
- }
- };
- alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
- assertTrue(alluxioCarbonFile.listFiles(carbonFileFilter).length == 1);
- }
-
- @Test
- public void testlistFilesWithoutFilter() {
- CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
-
- @Override
- public boolean accept(CarbonFile file) {
- return false;
- }
- };
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public FileStatus[] listStatus(Path var1) throws IOException {
-
- FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
- }
-
- };
- alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
- assertTrue(alluxioCarbonFile.listFiles(carbonFileFilter).length == 0);
- }
-
- @Test
- public void testGetParentFile() {
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
- new MockUp<Path>() {
- @Mock
- public Path getParent() {
- return new Path(file.getAbsolutePath()
- );
- }
-
- };
- new MockUp<FileStatus>() {
- @Mock
- public Path getPath() {
- return new Path(file.getAbsolutePath());
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public FileStatus getFileStatus(Path path) throws IOException {
-
- return new FileStatus(12L, true, 60, 120l, 180L, new Path(file.getAbsolutePath()));
- }
-
- };
-
- alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
- assertFalse(alluxioCarbonFile.getParentFile().equals(null));
- }
-
- @Test
- public void testForNonDisributedSystem() {
- alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new WebHdfsFileSystem();
- }
-
- };
- assertFalse(alluxioCarbonFile.renameForce(fileName));
- }
-
- @Test
- public void testrenameForceForDisributedSystem() {
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public void rename(Path src, Path dst, final Options.Rename... options) throws IOException {
-
- }
-
- };
-
- alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
- assertTrue(alluxioCarbonFile.renameForce(fileName));
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/datastorage/store/filesystem/HDFSCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/store/filesystem/HDFSCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/store/filesystem/HDFSCarbonFileTest.java
deleted file mode 100644
index ffaace1..0000000
--- a/core/src/test/java/org/apache/carbondata/core/datastorage/store/filesystem/HDFSCarbonFileTest.java
+++ /dev/null
@@ -1,402 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.filesystem;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastorage.store.filesystem.HDFSCarbonFile;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Options;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
-import org.apache.hadoop.util.Progressable;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.BufferedWriter;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-
-public class HDFSCarbonFileTest {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(HDFSCarbonFile.class.getName());
- private static HDFSCarbonFile hdfsCarbonFile;
- private static FileStatus fileStatus = null;
- private static FileStatus fileStatusWithOutDirectoryPermission;
- private static String fileName = null;
- private static FileSystem fs = null;
- private static Path pt;
-
- @BeforeClass
- static public void setUp() throws IOException {
- Configuration config = new Configuration();
-//adding local hadoop configuration
- config.addResource(new Path("core-site.xml"));
- config.addResource(new Path("hdfs-site.xml"));
- fileName = "Test.carbondata"; //this path is HDFS path
- pt = new Path(fileName);
- fs = FileSystem.get(new Configuration(config));
- fs.create(pt);
- if (fs.exists(pt)) {
- OutputStream os = fs.create(pt,
- new Progressable() {
- public void progress() {
- LOGGER.info("Started Writing to File===");
- }
- });
- BufferedWriter br = new BufferedWriter(new OutputStreamWriter(os, "UTF-8"));
- br.write("Hello World");
- br.close();
- fs.close();
-
- fileStatus = new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName));
- fileStatusWithOutDirectoryPermission = new FileStatus(12L, false, 60, 120l, 180L, new Path(fileName));
- hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
-
- }
- }
-
- @AfterClass
- static public void cleanUp() {
- try {
- fs.delete(pt, true);
- } catch (IOException e) {
- LOGGER.error("Exception Occured" + e.getMessage());
- }
- }
-
- @Test
- public void testRenameForceForException() throws IOException {
-
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- throw new IOException();
- }
-
- };
- hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
- hdfsCarbonFile.renameForce(fileName);
- }
-
- @Test
- public void testListFilesWithOutDirectoryPermission() {
- hdfsCarbonFile = new HDFSCarbonFile(fileStatusWithOutDirectoryPermission);
- new MockUp<FileStatus>() {
- @Mock
- public boolean isDirectory() {
- return false;
- }
-
- };
-
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
-
- new MockUp<FileStatus>() {
- @Mock
- public Path getPath() {
- return new Path(fileName);
- }
-
- };
- assertEquals(hdfsCarbonFile.listFiles(), null);
- }
-
- @Test
- public void testConstructorWithFilePath() {
- hdfsCarbonFile = new HDFSCarbonFile(fileName);
- assertTrue(hdfsCarbonFile instanceof HDFSCarbonFile);
- }
-
- @Test
- public void testListFilesForNullListStatus() {
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public FileStatus[] listStatus(Path var1) throws IOException {
-
- return null;
- }
-
- };
- hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
- assertEquals(hdfsCarbonFile.listFiles().length, 0);
- }
-
- @Test
- public void testListDirectory() {
- hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public FileStatus[] listStatus(Path var1) throws IOException {
-
- FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
- }
-
- };
-
- assertEquals(hdfsCarbonFile.listFiles().length, 1);
- }
-
- @Test
- public void testListFilesForException() throws IOException {
- new HDFSCarbonFile(fileStatusWithOutDirectoryPermission);
-
- new MockUp<FileStatus>() {
- @Mock
- public Path getPath() {
- return new Path(fileName);
- }
-
- };
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- throw new IOException();
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public FileStatus[] listStatus(Path var1) throws IOException {
-
- throw new IOException();
- }
-
- };
- hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
- hdfsCarbonFile.listFiles();
- }
-
- @Test
- public void testListFilesWithCarbonFilter() {
- CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
-
- @Override
- public boolean accept(CarbonFile file) {
- return true;
- }
- };
- new MockUp<FileStatus>() {
- @Mock
- public boolean isDirectory() {
- return true;
- }
-
- };
-
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
-
- new MockUp<FileStatus>() {
- @Mock
- public Path getPath() {
- return new Path(fileName);
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public FileStatus[] listStatus(Path var1) throws IOException {
-
- FileStatus fileStatus[] = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
- }
-
- };
- hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
- assertEquals(hdfsCarbonFile.listFiles(carbonFileFilter).length, 1);
- }
-
- @Test
- public void testlistFilesWithoutFilter() {
- CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
-
- @Override
- public boolean accept(CarbonFile file) {
- return false;
- }
- };
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public FileStatus[] listStatus(Path var1) throws IOException {
-
- FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
- }
-
- };
- hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
- assertEquals(hdfsCarbonFile.listFiles(carbonFileFilter).length, 0);
- }
-
- @Test
- public void testGetParentFileForNull() {
-
- new MockUp<Path>() {
- @Mock
- public Path getParent() {
- return null;
- }
-
- };
- new MockUp<FileStatus>() {
- @Mock
- public Path getPath() {
- return new Path(fileName);
- }
-
- };
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
-
- new MockUp<FileStatus>() {
- @Mock
- public Path getPath() {
- return new Path(fileName);
- }
-
- };
- hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
- assertEquals(hdfsCarbonFile.getParentFile(), null);
- }
-
- @Test
- public void testGetParentFile() {
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
- new MockUp<Path>() {
- @Mock
- public Path getParent() {
- return new Path(fileName);
- }
-
- };
- new MockUp<FileStatus>() {
- @Mock
- public Path getPath() {
- return new Path(fileName);
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public FileStatus getFileStatus(Path file) throws IOException {
-
- return new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName));
- }
-
- };
-
- hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
- assertTrue(hdfsCarbonFile.getParentFile() instanceof CarbonFile);
- }
-
- @Test
- public void testForNonDisributedSystem() {
- new HDFSCarbonFile(fileStatus);
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new WebHdfsFileSystem();
- }
-
- };
- assertEquals(hdfsCarbonFile.renameForce(fileName), false);
- }
-
- @Test
- public void testrenameForceForDisributedSystem() {
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public void rename(Path src, Path dst, final Options.Rename... options) throws IOException {
-
- }
-
- };
- hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
- assertEquals(hdfsCarbonFile.renameForce(fileName), true);
-
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/datastorage/store/filesystem/LocalCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/store/filesystem/LocalCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/store/filesystem/LocalCarbonFileTest.java
deleted file mode 100644
index 6ea4a9b..0000000
--- a/core/src/test/java/org/apache/carbondata/core/datastorage/store/filesystem/LocalCarbonFileTest.java
+++ /dev/null
@@ -1,472 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.filesystem;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastorage.store.filesystem.LocalCarbonFile;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import sun.nio.ch.FileChannelImpl;
-
-import java.io.*;
-import java.nio.channels.ReadableByteChannel;
-import java.util.Objects;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-
-public class LocalCarbonFileTest {
-
- private static LocalCarbonFile localCarbonFile;
- private static File file;
- private static File dir;
- private static FileOutputStream oFile;
-
- @BeforeClass
- static public void setUp() {
- file = new File("Test.carbondata");
- dir = new File("Testdir.carbondata");
- if (!file.exists())
- try {
- file.createNewFile();
- dir.mkdir();
- } catch (IOException e) {
- e.printStackTrace();
- }
- try {
- oFile = new FileOutputStream(file, true);
-
-
- byte[] bytes = "core java api".getBytes();
-
- oFile.write(bytes);
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- localCarbonFile = new LocalCarbonFile(file);
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- @AfterClass
- static public void cleanUp() {
- file.delete();
- dir.delete();
-
- }
-
- @Test
- public void testListFilesWithCarbonFileFilterAndWithOutOutDirectoryPermission() {
- CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
- @Override
- public boolean accept(CarbonFile file) {
- return false;
- }
- };
- new MockUp<File>() {
- @Mock
- public boolean isDirectory() {
- return false;
- }
-
-
- };
- assertTrue(localCarbonFile.listFiles(carbonFileFilter) == null);
- }
-
- @Test
- public void testListFilesWithOutDirPermission() {
- localCarbonFile = new LocalCarbonFile(file);
- new MockUp<File>() {
- @Mock
- public boolean isDirectory() {
- return false;
- }
- };
- assertTrue(localCarbonFile.listFiles() == null);
- }
-
- @Test
- public void testCreateNewFileForException() throws IOException {
- localCarbonFile = new LocalCarbonFile(new File(""));
- assertTrue(!localCarbonFile.createNewFile());
- }
-
- @Test
- public void testCheckIfFileExists() throws IOException {
- localCarbonFile = new LocalCarbonFile(new File(""));
- assertTrue(!localCarbonFile.exists());
- }
-
- @Test
- public void testRenameForce() {
- localCarbonFile = new LocalCarbonFile(file);
- assertTrue(localCarbonFile.renameForce("Testdb.carbon"));
- File file1 = new File("Testdb.carbon");
- if (file1.exists()) {
- file1.delete();
- }
- }
-
- @Test
- public void testRenameTo() {
- localCarbonFile = new LocalCarbonFile(file);
- assertTrue(!localCarbonFile.renameTo("Testdb.carbon"));
- }
-
- @Test
- public void testsetLastModifiedTime() {
- localCarbonFile = new LocalCarbonFile(file);
- assertTrue(!localCarbonFile.setLastModifiedTime(50L));
- }
-
- @Test
- public void testtruncate() {
- localCarbonFile = new LocalCarbonFile(file);
- final int[] counter = {0};
- new MockUp<FileFactory>() {
- @Mock
- public FileFactory.FileType getFileType(String path) {
- {
- return FileFactory.FileType.LOCAL;
- }
- }
- };
- new MockUp<FileFactory>() {
- @Mock
- public boolean isFileExist(String filePath, FileFactory.FileType fileType) throws IOException {
- {
- return true;
- }
- }
- };
- new MockUp<CarbonFile>() {
- @Mock
- boolean delete() {
- return true;
- }
- };
- new MockUp<FileFactory>() {
- @Mock
- public boolean createNewFile(String filePath, FileFactory.FileType fileType) throws IOException {
- {
- return true;
- }
- }
- };
- new MockUp<FileFactory>() {
- @Mock
- public CarbonFile getCarbonFile(String path, FileFactory.FileType fileType) {
- {
- return new LocalCarbonFile(path);
- }
- }
- };
- new MockUp<CarbonFile>() {
- @Mock
- boolean delete() {
- return true;
- }
- };
-
- new MockUp<FileChannelImpl>() {
- @Mock
- public long transferFrom(ReadableByteChannel var1, long var2, long var4) throws IOException {
- if (counter[0] == 0) {
- counter[0] = counter[0] + 1;
- return 0L;
- } else {
- return 1L;
- }
- }
- };
- new MockUp<CarbonFile>() {
- @Mock
- boolean renameForce(String changetoName) {
- return true;
- }
- };
- localCarbonFile = new LocalCarbonFile(file);
- assertTrue(localCarbonFile.truncate(file.getName(), 1L));
- }
-
- @Test
- public void testtruncateForException() throws IOException {
- localCarbonFile = new LocalCarbonFile(file);
- new MockUp<FileFactory>() {
- @Mock
- public FileFactory.FileType getFileType(String path) {
- {
- return FileFactory.FileType.LOCAL;
- }
- }
- };
- new MockUp<FileFactory>() {
- @Mock
- public boolean isFileExist(String filePath, FileFactory.FileType fileType) throws IOException {
- {
- return true;
- }
- }
- };
- new MockUp<FileFactory>() {
- @Mock
- public CarbonFile getCarbonFile(String path, FileFactory.FileType fileType) {
- {
- return new LocalCarbonFile(path);
- }
- }
- };
- new MockUp<CarbonFile>() {
- @Mock
- boolean delete() {
- return true;
- }
- };
- new MockUp<FileFactory>() {
- @Mock
- public boolean createNewFile(String filePath, FileFactory.FileType fileType) throws IOException {
- {
- throw new IOException();
- }
- }
- };
-
-
- localCarbonFile.truncate(file.getName(), 2L);
- }
-
- @Test
- public void testListFilesWithDirPermission() {
- localCarbonFile = new LocalCarbonFile(file);
- new MockUp<File>() {
- @Mock
- public boolean isDirectory() {
- return true;
- }
- };
- new MockUp<File>() {
- @Mock
- public File[] listFiles() {
- return null;
- }
-
-
- };
- localCarbonFile = new LocalCarbonFile(dir);
- assertTrue(localCarbonFile.listFiles().length == 0);
- }
-
- @Test
- public void testListFilesWithCarbonFileFilterAndDirectoryPermission() {
- CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
- @Override
- public boolean accept(CarbonFile file) {
- return true;
- }
- };
- new MockUp<File>() {
- @Mock
- public boolean isDirectory() {
- return true;
- }
- };
- new MockUp<File>() {
- @Mock
- public File[] listFiles(FileFilter filter) {
-
- File[] file = new File[]{dir};
- return file;
- }
-
-
- };
-
- localCarbonFile = new LocalCarbonFile(dir);
-
- assertTrue(localCarbonFile.listFiles(carbonFileFilter).length == 1);
- }
-
- @Test
- public void testListFilesForNullWithCarbonFileFilterAndDirectoryPermission() {
- CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
- @Override
- public boolean accept(CarbonFile file) {
- return true;
- }
- };
- new MockUp<File>() {
- @Mock
- public File[] listFiles(FileFilter filter) {
- return null;
- }
-
-
- };
- new MockUp<File>() {
- @Mock
- public boolean isDirectory() {
- return false;
- }
-
- @Mock
- public File[] listFiles(FileFilter filter) {
- return null;
- }
-
-
- };
- localCarbonFile = new LocalCarbonFile(dir);
-
- assertTrue(localCarbonFile.listFiles(carbonFileFilter) == null);
- }
-
- @Test
- public void testListFilesForEmptyFileArrayWithCarbonFileFilter() {
- CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
- @Override
- public boolean accept(CarbonFile file) {
- return true;
- }
- };
- new MockUp<CarbonFileFilter>() {
- @Mock
- boolean accept(CarbonFile file) {
- return true;
- }
- };
- new MockUp<File>() {
- @Mock
- public File[] listFiles(FileFilter filter) {
- return null;
- }
- };
- new MockUp<File>() {
- @Mock
- public boolean isDirectory() {
- return true;
- }
-
- @Mock
- public File[] listFiles(FileFilter filter) {
- return null;
- }
- };
- localCarbonFile = new LocalCarbonFile(dir);
-
- assertTrue(localCarbonFile.listFiles(carbonFileFilter).length == 0);
- }
-
- @Test
- public void testFilesForConnicalPath() {
-
- new MockUp<File>() {
- @Mock
- public String getCanonicalPath() throws IOException {
- return "testFile";
- }
-
-
- };
- localCarbonFile = new LocalCarbonFile(dir);
-
- assertTrue(Objects.equals(localCarbonFile.getCanonicalPath(), "testFile"));
- }
-
- @Test
- public void testFilesForConnicalPathException() throws IOException {
-
- new MockUp<File>() {
- @Mock
- public String getCanonicalPath() throws IOException {
- throw new IOException();
- }
-
-
- };
- localCarbonFile = new LocalCarbonFile(dir);
-
- localCarbonFile.getCanonicalPath();
- }
-
- @Test
- public void testFilesForAbsolutePath() {
-
- new MockUp<File>() {
- @Mock
- public String getAbsolutePath() {
- return "testFile";
- }
-
-
- };
- localCarbonFile = new LocalCarbonFile(dir);
-
- assertEquals(localCarbonFile.getAbsolutePath(), "testFile");
- }
-
- @Test
- public void testFilesForGetPath() {
-
- new MockUp<File>() {
- @Mock
- public String getPath() {
- return "testFile";
- }
-
-
- };
- localCarbonFile = new LocalCarbonFile(dir);
-
- assertEquals(localCarbonFile.getPath(), "testFile");
- }
-
- @Test
- public void testFilesForFileExists() {
-
- localCarbonFile = new LocalCarbonFile(new File(""));
- assertEquals(localCarbonFile.exists(), false);
- }
-
- @Test
- public void testRenameForceForFileNotExists() {
- new MockUp<File>() {
- @Mock
- public boolean exists() {
- return false;
- }
-
- @Mock
- public boolean renameTo(File dest) {
- return true;
- }
- };
-
- localCarbonFile = new LocalCarbonFile("demo.txt");
-
- assertEquals(localCarbonFile.renameForce("Test.carbondata"), true);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/datastorage/store/filesystem/ViewFsCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/store/filesystem/ViewFsCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/store/filesystem/ViewFsCarbonFileTest.java
deleted file mode 100644
index de37fc7..0000000
--- a/core/src/test/java/org/apache/carbondata/core/datastorage/store/filesystem/ViewFsCarbonFileTest.java
+++ /dev/null
@@ -1,314 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.filesystem;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastorage.store.filesystem.ViewFSCarbonFile;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.viewfs.ViewFileSystem;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-
-public class ViewFsCarbonFileTest {
-
- private static ViewFSCarbonFile viewFSCarbonFile;
- private static FileStatus fileStatus;
- private static FileStatus fileStatusWithOutDirectoryPermission;
- private static String fileName;
- private static File file;
-
-
- @BeforeClass
- static public void setUp() {
- file = new File("Test.carbondata");
- if (!file.exists())
- try {
- file.createNewFile();
- } catch (IOException e) {
- e.printStackTrace();
- }
- try {
- FileOutputStream oFile = new FileOutputStream(file, true);
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- }
-
- fileStatus = new FileStatus(12L, true, 60, 120l, 180L, new Path(file.getAbsolutePath()));
- fileStatusWithOutDirectoryPermission = new FileStatus(12L, false, 60, 120l, 180L, new Path(file.getAbsolutePath()));
- fileName = file.getAbsolutePath();
- viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
- }
-
- @AfterClass
- static public void cleanUp() {
- file.delete();
- }
-
- @Test
- public void testRenameForceForException() throws IOException {
-
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- throw new IOException();
- }
-
- };
- viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
- viewFSCarbonFile.renameForce(fileName);
- }
-
- @Test
- public void testListFilesWithOutDirectoryPermission() {
- viewFSCarbonFile = new ViewFSCarbonFile(fileStatusWithOutDirectoryPermission);
- assertTrue(viewFSCarbonFile.listFiles() == null);
- }
-
- @Test
- public void testConstructorWithFilePath() {
- viewFSCarbonFile = new ViewFSCarbonFile(file.getAbsolutePath());
- assertTrue(viewFSCarbonFile instanceof ViewFSCarbonFile);
- }
-
- @Test
- public void testListFilesForNullListStatus() {
- viewFSCarbonFile = new ViewFSCarbonFile(fileStatusWithOutDirectoryPermission);
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new ViewFileSystem();
- }
-
- };
- new MockUp<ViewFileSystem>() {
- @Mock
- public FileStatus[] listStatus(Path var1) throws IOException {
-
- return null;
- }
-
- };
- new MockUp<ViewFileSystem>() {
- @Mock
- public boolean delete(Path var1, boolean var2) throws IOException {
-
- return true;
- }
-
- };
- //public boolean delete(Path var1, boolean var2) throws IOException;
- viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
- assertTrue(viewFSCarbonFile.listFiles().length == 0);
- }
-
- @Test
- public void testListDirectory() {
- viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new ViewFileSystem();
- }
-
- };
- new MockUp<ViewFileSystem>() {
- @Mock
- public FileStatus[] listStatus(Path var1) throws IOException {
-
- FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
- }
-
- };
-
- assertTrue(viewFSCarbonFile.listFiles().length == 1);
- }
-
- @Test
- public void testListFilesForException() throws IOException {
- viewFSCarbonFile = new ViewFSCarbonFile(fileStatusWithOutDirectoryPermission);
-
- new MockUp<FileStatus>() {
- @Mock
- public Path getPath() {
- return new Path(file.getAbsolutePath());
- }
-
- };
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- throw new IOException();
- }
-
- };
- new MockUp<ViewFileSystem>() {
- @Mock
- public FileStatus[] listStatus(Path var1) throws IOException {
-
- throw new IOException();
- }
-
- };
- viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
- viewFSCarbonFile.listFiles();
- }
-
- @Test
- public void testListFilesWithCarbonFilter() {
- CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
-
- @Override
- public boolean accept(CarbonFile file) {
- return true;
- }
- };
- viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
- assertTrue(viewFSCarbonFile.listFiles(carbonFileFilter).length == 1);
- }
-
- @Test
- public void testlistFilesWithoutFilter() {
- CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
-
- @Override
- public boolean accept(CarbonFile file) {
- return false;
- }
- };
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new ViewFileSystem();
- }
-
- };
- new MockUp<ViewFileSystem>() {
- @Mock
- public FileStatus[] listStatus(Path var1) throws IOException {
-
- FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
- }
-
- };
- viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
- assertTrue(viewFSCarbonFile.listFiles(carbonFileFilter).length == 0);
- }
-
- @Test
- public void testGetParentFIle() {
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
- new MockUp<Path>() {
- @Mock
- public Path getParent() {
- return new Path(file.getAbsolutePath()
- );
- }
-
- };
- new MockUp<FileStatus>() {
- @Mock
- public Path getPath() {
- return new Path(file.getAbsolutePath());
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public FileStatus getFileStatus(Path f) throws IOException {
-
- return new FileStatus(12L, true, 60, 120l, 180L, new Path(file.getAbsolutePath()));
- }
-
- };
-
- viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
- assertFalse(viewFSCarbonFile.getParentFile().equals(null));
- }
-
- @Test
- public void testForNonDisributedSystem() {
- viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new WebHdfsFileSystem();
- }
-
- };
- assertFalse(viewFSCarbonFile.renameForce(fileName));
- }
-
- @Test
- public void testrenameForceForViewFileSystem() {
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new ViewFileSystem();
- }
-
- };
- new MockUp<ViewFileSystem>() {
- @Mock
- public boolean delete(Path f, boolean recursive) throws
- IOException {
- return true;
-
- }
-
- };
- new MockUp<ViewFileSystem>() {
- @Mock
- public boolean rename(Path src, Path dst) throws IOException {
- return true;
-
- }
-
- };
-
- assertTrue(viewFSCarbonFile.renameForce(fileName));
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunkTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunkTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunkTest.java
index 0fe8e63..7d7c7d0 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunkTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunkTest.java
@@ -30,8 +30,8 @@ import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.scan.executor.util.QueryUtil;
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.executor.util.QueryUtil;
import org.junit.BeforeClass;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java
index b5cf412..bd5ba96 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunkTest.java
@@ -23,7 +23,7 @@ import static org.junit.Assert.assertEquals;
import java.util.Arrays;
-import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.junit.BeforeClass;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReaderTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReaderTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReaderTest.java
index 4cb8cc6..29fe16d 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReaderTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReaderTest.java
@@ -1,29 +1,31 @@
package org.apache.carbondata.core.datastore.chunk.reader.measure;
-import static junit.framework.TestCase.assertEquals;
-
import java.io.IOException;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.List;
-import mockit.Mock;
-import mockit.MockUp;
-
+import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.datastorage.compression.MeasureMetaDataModel;
+import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder;
+import org.apache.carbondata.core.datastorage.impl.data.compressed
+ .HeavyCompressedDoubleArrayDataInMemoryStore;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.reader.measure.v1.CompressedMeasureChunkFileBasedReaderV1;
+import org.apache.carbondata.core.datastore.chunk.reader.measure
+ .v1.CompressedMeasureChunkFileBasedReaderV1;
+import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk;
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.core.datastorage.store.compression.MeasureMetaDataModel;
-import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder;
-import org.apache.carbondata.core.datastorage.util.StoreFactory;
-import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.core.util.ValueCompressionUtil;
+
+import mockit.Mock;
+import mockit.MockUp;
import org.junit.BeforeClass;
import org.junit.Test;
+import static junit.framework.TestCase.assertEquals;
+
public class CompressedMeasureChunkFileBasedReaderTest {
static CompressedMeasureChunkFileBasedReaderV1 compressedMeasureChunkFileBasedReader;
@@ -67,7 +69,7 @@ public class CompressedMeasureChunkFileBasedReaderTest {
dataHolder[0].initialiseBigDecimalValues(1);
dataHolder[0].setWritableBigDecimalValueByIndex(0, new long[]{2L,1L});
byte[][] writableMeasureDataArray =
- StoreFactory.createDataStore(writerCompressModel).getWritableMeasureDataArray(dataHolder)
+ new HeavyCompressedDoubleArrayDataInMemoryStore(writerCompressModel).getWritableMeasureDataArray(dataHolder)
.clone();
return writableMeasureDataArray[0];
}
@@ -89,7 +91,7 @@ public class CompressedMeasureChunkFileBasedReaderTest {
dataHolder[0].initialiseBigDecimalValues(1);
dataHolder[0].setWritableBigDecimalValueByIndex(0, new long[]{2L,1L});
byte[][] writableMeasureDataArray =
- StoreFactory.createDataStore(writerCompressModel).getWritableMeasureDataArray(dataHolder)
+ new HeavyCompressedDoubleArrayDataInMemoryStore(writerCompressModel).getWritableMeasureDataArray(dataHolder)
.clone();
return writableMeasureDataArray[0];
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImplTest.java b/core/src/test/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImplTest.java
index b8a97c4..59ac840 100644
--- a/core/src/test/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImplTest.java
@@ -25,8 +25,8 @@ import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
-import org.apache.carbondata.common.ext.PathFactory;
-import org.apache.carbondata.common.factory.CarbonCommonFactory;
+import org.apache.carbondata.core.service.impl.PathFactory;
+import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.core.CarbonTableIdentifier;
import org.apache.carbondata.core.ColumnIdentifier;
import org.apache.carbondata.core.service.PathService;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java b/core/src/test/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java
index 9b62235..e1e0e94 100644
--- a/core/src/test/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java
@@ -26,8 +26,8 @@ import java.util.UUID;
import org.apache.carbondata.core.CarbonTableIdentifier;
import org.apache.carbondata.core.ColumnIdentifier;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.writer.CarbonDictionaryWriter;
import org.apache.carbondata.core.writer.CarbonDictionaryWriterImpl;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollectorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollectorTest.java b/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollectorTest.java
new file mode 100644
index 0000000..1b2f533
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollectorTest.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.collector.impl;
+
+
+public class DictionaryBasedResultCollectorTest {
+
+// private static DictionaryBasedResultCollector dictionaryBasedResultCollector;
+// private static BlockExecutionInfo blockExecutionInfo;
+//
+// @BeforeClass public static void setUp() {
+// blockExecutionInfo = new BlockExecutionInfo();
+// KeyStructureInfo keyStructureInfo = new KeyStructureInfo();
+// blockExecutionInfo.setKeyStructureInfo(keyStructureInfo);
+// AggregatorInfo aggregatorInfo = new AggregatorInfo();
+// aggregatorInfo.setMeasureOrdinals(new int[] { 10, 20, 30, 40 });
+// aggregatorInfo.setMeasureExists(new boolean[] { true, false, false, false });
+// aggregatorInfo.setDefaultValues(new Object[] { 1, 2, 3, 4 });
+// aggregatorInfo.setMeasureDataTypes(
+// new DataType[] { DataType.INT, DataType.TIMESTAMP, DataType.INT, DataType.INT });
+// blockExecutionInfo.setAggregatorInfo(aggregatorInfo);
+// QueryDimension queryDimension1 = new QueryDimension("QDCol1");
+// queryDimension1.setQueryOrder(1);
+// ColumnSchema columnSchema = new ColumnSchema();
+// queryDimension1.setDimension(new CarbonDimension(columnSchema, 0, 0, 0, 0));
+// QueryDimension queryDimension2 = new QueryDimension("QDCol2");
+// queryDimension2.setQueryOrder(2);
+// queryDimension2.setDimension(new CarbonDimension(columnSchema, 1, 1, 1, 1));
+// QueryDimension queryDimension3 = new QueryDimension("QDCol3");
+// queryDimension3.setQueryOrder(3);
+// queryDimension3.setDimension(new CarbonDimension(columnSchema, 2, 0, 0, 0));
+// QueryDimension queryDimension4 = new QueryDimension("QDCol4");
+// queryDimension4.setQueryOrder(4);
+// queryDimension4.setDimension(new CarbonDimension(columnSchema, 3, 0, 0, 0));
+// blockExecutionInfo.setQueryDimensions(
+// new QueryDimension[] { queryDimension1, queryDimension2, queryDimension3,
+// queryDimension4 });
+// QueryMeasure queryMeasure1 = new QueryMeasure("QMCol1");
+// queryMeasure1.setQueryOrder(1);
+// QueryMeasure queryMeasure2 = new QueryMeasure("QMCol2");
+// queryMeasure1.setQueryOrder(2);
+// QueryMeasure queryMeasure3 = new QueryMeasure("QMCol3");
+// queryMeasure1.setQueryOrder(3);
+// QueryMeasure queryMeasure4 = new QueryMeasure("QMCol4");
+// queryMeasure1.setQueryOrder(4);
+// blockExecutionInfo.setQueryMeasures(
+// new QueryMeasure[] { queryMeasure1, queryMeasure2, queryMeasure3, queryMeasure4 });
+// Map<Integer, GenericQueryType> complexDimensionInfoMap = new HashMap<>();
+// complexDimensionInfoMap.put(1, new ArrayQueryType("name1", "parent1", 1));
+// complexDimensionInfoMap.put(2, new ArrayQueryType("name2", "parent2", 2));
+// complexDimensionInfoMap.put(3, new ArrayQueryType("name3", "parent3", 3));
+// complexDimensionInfoMap.put(4, new ArrayQueryType("name4", "parent4", 4));
+// blockExecutionInfo.setComplexDimensionInfoMap(complexDimensionInfoMap);
+// dictionaryBasedResultCollector = new DictionaryBasedResultCollector(blockExecutionInfo);
+// }
+//
+// @Test public void testToCollectData() {
+// new MockUp<CarbonUtil>() {
+// @SuppressWarnings("unused") @Mock boolean[] getDictionaryEncodingArray(
+// QueryDimension[] queryDimensions) {
+// return new boolean[] { true, false, true, true };
+// }
+//
+// @SuppressWarnings("unused") @Mock boolean[] getDirectDictionaryEncodingArray(
+// QueryDimension[] queryDimensions) {
+// return new boolean[] { true, true, false, false };
+// }
+//
+// @SuppressWarnings("unused") @Mock boolean[] getComplexDataTypeArray(
+// QueryDimension[] queryDimensions) {
+// return new boolean[] { false, false, true, false };
+// }
+// };
+// new MockUp<DataTypeUtil>() {
+// @SuppressWarnings("unused") @Mock Object getDataBasedOnDataType(String data,
+// DataType actualDataType) {
+// return 1;
+// }
+// };
+//
+// new MockUp<NonFilterQueryScannedResult>() {
+// @SuppressWarnings("unused") @Mock int[] getDictionaryKeyIntegerArray() {
+// this.getMockInstance().incrementCounter();
+// System.out.println("Mocked");
+// return new int[] { 1, 2 };
+// }
+//
+// @SuppressWarnings("unused") @Mock String[] getNoDictionaryKeyStringArray() {
+// return new String[] { "1", "2" };
+// }
+//
+// @SuppressWarnings("unused") @Mock byte[][] getComplexTypeKeyArray() {
+// return new byte[][] { { 1, 2 }, { 1, 2 } };
+// }
+//
+// @SuppressWarnings("unused") @Mock public MeasureColumnDataChunk getMeasureChunk(int ordinal) {
+// MeasureColumnDataChunk measureColumnDataChunk = new MeasureColumnDataChunk();
+// PresenceMeta presenceMeta = new PresenceMeta();
+// BitSet bitSet = new BitSet();
+// bitSet.set(1);
+// presenceMeta.setBitSet(bitSet);
+// measureColumnDataChunk.setNullValueIndexHolder(presenceMeta);
+// CarbonReadDataHolder carbonReadDataHolder = new CarbonReadDataHolder();
+// carbonReadDataHolder.setReadableLongValues(new long[] { 1 });
+// measureColumnDataChunk.setMeasureDataHolder(carbonReadDataHolder);
+// return measureColumnDataChunk;
+// }
+// };
+//
+// new MockUp<DirectDictionaryKeyGeneratorFactory>() {
+// @SuppressWarnings("unused") @Mock DirectDictionaryGenerator getDirectDictionaryGenerator(
+// DataType dataType) {
+// if (dataType == DataType.TIMESTAMP) return new TimeStampDirectDictionaryGenerator(
+// CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT);
+// else return null;
+// }
+// };
+// new MockUp<TimeStampDirectDictionaryGenerator>() {
+// @SuppressWarnings("unused") @Mock Object getValueFromSurrogate(int key) {
+// return 100L;
+// }
+// };
+//
+// new MockUp<ArrayQueryType>() {
+// @SuppressWarnings("unused") @Mock Object getDataBasedOnDataTypeFromSurrogates(
+// ByteBuffer surrogateData) {
+// return ByteBuffer.wrap("1".getBytes());
+// }
+// };
+//
+// AbstractScannedResult abstractScannedResult =
+// new NonFilterQueryScannedResult(blockExecutionInfo);
+// abstractScannedResult.setNumberOfRows(2);
+// List<Object[]> result = dictionaryBasedResultCollector.collectData(abstractScannedResult, 2);
+// int expectedResult = 2;
+// assertThat(result.size(), is(equalTo(expectedResult)));
+// }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollectorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollectorTest.java b/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollectorTest.java
new file mode 100644
index 0000000..a87e272
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollectorTest.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.collector.impl;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+public class RawBasedResultCollectorTest {
+//
+// private static RawBasedResultCollector rawBasedResultCollector;
+// private static BlockExecutionInfo blockExecutionInfo;
+// private static KeyGenerator keyGenerator;
+//
+// @BeforeClass public static void setUp() {
+// keyGenerator = new MockUp<KeyGenerator>() {
+// @SuppressWarnings("unused") @Mock long[] getKeyArray(byte[] key, int[] maskedByteRanges) {
+// return new long[] { 1, 2 };
+// }
+//
+// @SuppressWarnings("unused") @Mock byte[] generateKey(long[] keys) throws KeyGenException {
+// return new byte[] { 1, 2 };
+// }
+//
+// }.getMockInstance();
+//
+// blockExecutionInfo = new BlockExecutionInfo();
+// KeyStructureInfo keyStructureInfo = new KeyStructureInfo();
+// keyStructureInfo.setKeyGenerator(keyGenerator);
+// keyStructureInfo.setMaxKey(new byte[] { 1, 2 });
+// keyStructureInfo.setMaskedBytes(new int[] { 1, 2 });
+// keyStructureInfo.setMaskByteRanges(new int[] { 1, 2 });
+// blockExecutionInfo.setKeyStructureInfo(keyStructureInfo);
+// AggregatorInfo aggregatorInfo = new AggregatorInfo();
+// aggregatorInfo.setMeasureOrdinals(new int[] { 10, 20, 30, 40 });
+// aggregatorInfo.setMeasureExists(new boolean[] { true, false, false, false });
+// aggregatorInfo.setDefaultValues(new Object[] { 1, 2, 3, 4 });
+// aggregatorInfo.setMeasureDataTypes(
+// new DataType[] { DataType.INT, DataType.TIMESTAMP, DataType.INT, DataType.INT });
+// QueryMeasure queryMeasure1 = new QueryMeasure("QMCol1");
+// queryMeasure1.setQueryOrder(1);
+// QueryMeasure queryMeasure2 = new QueryMeasure("QMCol2");
+// queryMeasure1.setQueryOrder(2);
+// QueryMeasure queryMeasure3 = new QueryMeasure("QMCol3");
+// queryMeasure1.setQueryOrder(3);
+// QueryMeasure queryMeasure4 = new QueryMeasure("QMCol4");
+// queryMeasure1.setQueryOrder(4);
+// QueryDimension queryDimension1 = new QueryDimension("QDCol1");
+// queryDimension1.setQueryOrder(1);
+// ColumnSchema columnSchema = new ColumnSchema();
+// queryDimension1.setDimension(new CarbonDimension(columnSchema, 0, 0, 0, 0));
+// QueryDimension queryDimension2 = new QueryDimension("QDCol2");
+// queryDimension2.setQueryOrder(2);
+// queryDimension2.setDimension(new CarbonDimension(columnSchema, 1, 1, 1, 1));
+// QueryDimension queryDimension3 = new QueryDimension("QDCol3");
+// queryDimension3.setQueryOrder(3);
+// queryDimension3.setDimension(new CarbonDimension(columnSchema, 2, 0, 0, 0));
+// QueryDimension queryDimension4 = new QueryDimension("QDCol4");
+// queryDimension4.setQueryOrder(4);
+// queryDimension4.setDimension(new CarbonDimension(columnSchema, 3, 0, 0, 0));
+// blockExecutionInfo.setQueryDimensions(
+// new QueryDimension[] { queryDimension1, queryDimension2, queryDimension3,
+// queryDimension4 });
+// blockExecutionInfo.setQueryMeasures(
+// new QueryMeasure[] { queryMeasure1, queryMeasure2, queryMeasure3, queryMeasure4 });
+// blockExecutionInfo.setFixedKeyUpdateRequired(true);
+// blockExecutionInfo.setAggregatorInfo(aggregatorInfo);
+// blockExecutionInfo.setMaskedByteForBlock(new int[] { 1, 2 });
+// blockExecutionInfo.setBlockKeyGenerator(keyGenerator);
+// rawBasedResultCollector = new RawBasedResultCollector(blockExecutionInfo);
+// }
+//
+// @Test public void testToCollectData() {
+//
+// new MockUp<NonFilterQueryScannedResult>() {
+// @SuppressWarnings("unused") @Mock byte[] getDictionaryKeyArray() {
+// this.getMockInstance().incrementCounter();
+// return new byte[] { 1, 2 };
+// }
+//
+// @SuppressWarnings("unused") @Mock byte[][] getNoDictionaryKeyArray() {
+// return new byte[][] { { 1, 2 } };
+// }
+//
+// @SuppressWarnings("unused") @Mock byte[][] getComplexTypeKeyArray() {
+// return new byte[][] { { 1, 2 }, { 1, 2 } };
+// }
+//
+// @SuppressWarnings("unused") @Mock public MeasureColumnDataChunk getMeasureChunk(int ordinal) {
+// MeasureColumnDataChunk measureColumnDataChunk = new MeasureColumnDataChunk();
+// PresenceMeta presenceMeta = new PresenceMeta();
+// BitSet bitSet = new BitSet();
+// bitSet.set(1);
+// presenceMeta.setBitSet(bitSet);
+// measureColumnDataChunk.setNullValueIndexHolder(presenceMeta);
+// CarbonReadDataHolder carbonReadDataHolder = new CarbonReadDataHolder();
+// carbonReadDataHolder.setReadableLongValues(new long[] { 1 });
+// measureColumnDataChunk.setMeasureDataHolder(carbonReadDataHolder);
+// return measureColumnDataChunk;
+// }
+// };
+//
+// new MockUp<QueryUtil>() {
+// @SuppressWarnings("unused") @Mock byte[] getMaskedKey(byte[] data, byte[] maxKey,
+// int[] maskByteRanges, int byteCount) {
+// return new byte[] { 1, 2 };
+// }
+// };
+//
+// AbstractScannedResult abstractScannedResult =
+// new NonFilterQueryScannedResult(blockExecutionInfo);
+// abstractScannedResult.setNumberOfRows(2);
+// List<Object[]> result = rawBasedResultCollector.collectData(abstractScannedResult, 2);
+// int expectedResult = 2;
+// assertThat(result.size(), is(equalTo(expectedResult)));
+// }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryTypeTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryTypeTest.java b/core/src/test/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryTypeTest.java
new file mode 100644
index 0000000..a7b31e3
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryTypeTest.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.complextypes;
+
+import java.nio.ByteBuffer;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.pentaho.di.core.util.Assert.assertNotNull;
+import static org.pentaho.di.core.util.Assert.assertNull;
+
+public class ArrayQueryTypeTest {
+
+ private static ArrayQueryType arrayQueryType;
+
+ @BeforeClass public static void setUp() {
+ String name = "test";
+ String parentName = "testParent";
+ int blockIndex = 1;
+ arrayQueryType = new ArrayQueryType(name, parentName, blockIndex);
+ }
+
+ @Test public void testGetDataBasedOnDataTypeFromSurrogatesForNull() {
+ ByteBuffer surrogateData = ByteBuffer.allocate(10);
+ surrogateData.put(0, (byte) 0xFF);
+ surrogateData.put(1, (byte) 0xFF);
+ surrogateData.put(2, (byte) 0xFF);
+ surrogateData.put(3, (byte) 0xFF);
+ assertNull(arrayQueryType.getDataBasedOnDataTypeFromSurrogates(surrogateData));
+ }
+
+ @Test public void testGetDataBasedOnDataTypeFromSurrogates() {
+ ByteBuffer surrogateData = ByteBuffer.allocate(10);
+ surrogateData.put(3, (byte) 1);
+ arrayQueryType.setName("testName");
+ arrayQueryType.setParentname("testName");
+ arrayQueryType.addChildren(arrayQueryType);
+ assertNotNull(arrayQueryType.getDataBasedOnDataTypeFromSurrogates(surrogateData));
+ }
+}
[31/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/LocalCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/LocalCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/LocalCarbonFile.java
new file mode 100644
index 0000000..d7866c1
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/LocalCarbonFile.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.filesystem;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+import org.apache.hadoop.fs.Path;
+
+public class LocalCarbonFile implements CarbonFile {
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(LocalCarbonFile.class.getName());
+ private File file;
+
+ public LocalCarbonFile(String filePath) {
+ Path pathWithoutSchemeAndAuthority = Path.getPathWithoutSchemeAndAuthority(new Path(filePath));
+ file = new File(pathWithoutSchemeAndAuthority.toString());
+ }
+
+ public LocalCarbonFile(File file) {
+ this.file = file;
+ }
+
+ @Override public String getAbsolutePath() {
+ return file.getAbsolutePath();
+ }
+
+ @Override public CarbonFile[] listFiles(final CarbonFileFilter fileFilter) {
+ if (!file.isDirectory()) {
+ return null;
+ }
+
+ File[] files = file.listFiles(new FileFilter() {
+
+ @Override public boolean accept(File pathname) {
+ return fileFilter.accept(new LocalCarbonFile(pathname));
+ }
+ });
+
+ if (files == null) {
+ return new CarbonFile[0];
+ }
+
+ CarbonFile[] carbonFiles = new CarbonFile[files.length];
+
+ for (int i = 0; i < carbonFiles.length; i++) {
+ carbonFiles[i] = new LocalCarbonFile(files[i]);
+ }
+
+ return carbonFiles;
+ }
+
+ @Override public String getName() {
+ return file.getName();
+ }
+
+ @Override public boolean isDirectory() {
+ return file.isDirectory();
+ }
+
+ @Override public boolean exists() {
+ if (file != null) {
+ return file.exists();
+ }
+ return false;
+ }
+
+ @Override public String getCanonicalPath() {
+ try {
+ return file.getCanonicalPath();
+ } catch (IOException e) {
+ LOGGER
+ .error(e, "Exception occured" + e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public CarbonFile getParentFile() {
+ return new LocalCarbonFile(file.getParentFile());
+ }
+
+ @Override public String getPath() {
+ return file.getPath();
+ }
+
+ @Override public long getSize() {
+ return file.length();
+ }
+
+ public boolean renameTo(String changetoName) {
+ return file.renameTo(new File(changetoName));
+ }
+
+ public boolean delete() {
+ return file.delete();
+ }
+
+ @Override public CarbonFile[] listFiles() {
+
+ if (!file.isDirectory()) {
+ return null;
+ }
+ File[] files = file.listFiles();
+ if (files == null) {
+ return new CarbonFile[0];
+ }
+ CarbonFile[] carbonFiles = new CarbonFile[files.length];
+ for (int i = 0; i < carbonFiles.length; i++) {
+ carbonFiles[i] = new LocalCarbonFile(files[i]);
+ }
+
+ return carbonFiles;
+
+ }
+
+ @Override public boolean createNewFile() {
+ try {
+ return file.createNewFile();
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ @Override public long getLastModifiedTime() {
+ return file.lastModified();
+ }
+
+ @Override public boolean setLastModifiedTime(long timestamp) {
+ return file.setLastModified(timestamp);
+ }
+
+ /**
+ * This method will delete the data in file data from a given offset
+ */
+ @Override public boolean truncate(String fileName, long validDataEndOffset) {
+ FileChannel source = null;
+ FileChannel destination = null;
+ boolean fileTruncatedSuccessfully = false;
+ // temporary file name
+ String tempWriteFilePath = fileName + CarbonCommonConstants.TEMPWRITEFILEEXTENSION;
+ FileFactory.FileType fileType = FileFactory.getFileType(fileName);
+ try {
+ CarbonFile tempFile = null;
+ // delete temporary file if it already exists at a given path
+ if (FileFactory.isFileExist(tempWriteFilePath, fileType)) {
+ tempFile = FileFactory.getCarbonFile(tempWriteFilePath, fileType);
+ tempFile.delete();
+ }
+ // create new temporary file
+ FileFactory.createNewFile(tempWriteFilePath, fileType);
+ tempFile = FileFactory.getCarbonFile(tempWriteFilePath, fileType);
+ source = new FileInputStream(fileName).getChannel();
+ destination = new FileOutputStream(tempWriteFilePath).getChannel();
+ long read = destination.transferFrom(source, 0, validDataEndOffset);
+ long totalBytesRead = read;
+ long remaining = validDataEndOffset - totalBytesRead;
+ // read till required data offset is not reached
+ while (remaining > 0) {
+ read = destination.transferFrom(source, totalBytesRead, remaining);
+ totalBytesRead = totalBytesRead + read;
+ remaining = remaining - totalBytesRead;
+ }
+ CarbonUtil.closeStreams(source, destination);
+ // rename the temp file to original file
+ tempFile.renameForce(fileName);
+ fileTruncatedSuccessfully = true;
+ } catch (IOException e) {
+ LOGGER.error("Exception occured while truncating the file " + e.getMessage());
+ } finally {
+ CarbonUtil.closeStreams(source, destination);
+ }
+ return fileTruncatedSuccessfully;
+ }
+
+ /**
+ * This method will be used to check whether a file has been modified or not
+ *
+ * @param fileTimeStamp time to be compared with latest timestamp of file
+ * @param endOffset file length to be compared with current length of file
+ * @return
+ */
+ @Override public boolean isFileModified(long fileTimeStamp, long endOffset) {
+ boolean isFileModified = false;
+ if (getLastModifiedTime() > fileTimeStamp || getSize() > endOffset) {
+ isFileModified = true;
+ }
+ return isFileModified;
+ }
+
+ @Override public boolean renameForce(String changetoName) {
+ File destFile = new File(changetoName);
+ if (destFile.exists()) {
+ if (destFile.delete()) {
+ return file.renameTo(new File(changetoName));
+ }
+ }
+
+ return file.renameTo(new File(changetoName));
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/ViewFSCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/ViewFSCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/ViewFSCarbonFile.java
new file mode 100644
index 0000000..3fcf387
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/ViewFSCarbonFile.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.datastorage.filesystem;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.viewfs.ViewFileSystem;
+
+public class ViewFSCarbonFile extends AbstractDFSCarbonFile {
+ /**
+ * LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(ViewFSCarbonFile.class.getName());
+
+ public ViewFSCarbonFile(String filePath) {
+ super(filePath);
+ }
+
+ public ViewFSCarbonFile(Path path) {
+ super(path);
+ }
+
+ public ViewFSCarbonFile(FileStatus fileStatus) {
+ super(fileStatus);
+ }
+
+ /**
+ * @param listStatus
+ * @return
+ */
+ private CarbonFile[] getFiles(FileStatus[] listStatus) {
+ if (listStatus == null) {
+ return new CarbonFile[0];
+ }
+ CarbonFile[] files = new CarbonFile[listStatus.length];
+ for (int i = 0; i < files.length; i++) {
+ files[i] = new ViewFSCarbonFile(listStatus[i]);
+ }
+ return files;
+ }
+
+ @Override
+ public CarbonFile[] listFiles() {
+ FileStatus[] listStatus = null;
+ try {
+ if (null != fileStatus && fileStatus.isDirectory()) {
+ Path path = fileStatus.getPath();
+ listStatus = path.getFileSystem(FileFactory.getConfiguration()).listStatus(path);
+ } else {
+ return null;
+ }
+ } catch (IOException ex) {
+ LOGGER.error("Exception occured" + ex.getMessage());
+ return new CarbonFile[0];
+ }
+ return getFiles(listStatus);
+ }
+
+ @Override
+ public CarbonFile[] listFiles(final CarbonFileFilter fileFilter) {
+ CarbonFile[] files = listFiles();
+ if (files != null && files.length >= 1) {
+ List<CarbonFile> fileList = new ArrayList<CarbonFile>(files.length);
+ for (int i = 0; i < files.length; i++) {
+ if (fileFilter.accept(files[i])) {
+ fileList.add(files[i]);
+ }
+ }
+ if (fileList.size() >= 1) {
+ return fileList.toArray(new CarbonFile[fileList.size()]);
+ } else {
+ return new CarbonFile[0];
+ }
+ }
+ return files;
+ }
+
+ @Override public CarbonFile getParentFile() {
+ Path parent = fileStatus.getPath().getParent();
+ return null == parent ? null : new ViewFSCarbonFile(parent);
+ }
+
+ @Override
+ public boolean renameForce(String changetoName) {
+ FileSystem fs;
+ try {
+ fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
+ if (fs instanceof ViewFileSystem) {
+ fs.delete(new Path(changetoName), true);
+ fs.rename(fileStatus.getPath(), new Path(changetoName));
+ return true;
+ } else {
+ return false;
+ }
+ } catch (IOException e) {
+ LOGGER.error("Exception occured" + e.getMessage());
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/impl/CompressedDataMeasureDataWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/CompressedDataMeasureDataWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/CompressedDataMeasureDataWrapper.java
new file mode 100644
index 0000000..6512d25
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/CompressedDataMeasureDataWrapper.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.impl;
+
+import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.datastorage.MeasureDataWrapper;
+
+public class CompressedDataMeasureDataWrapper implements MeasureDataWrapper {
+
+ private final CarbonReadDataHolder[] values;
+
+ public CompressedDataMeasureDataWrapper(final CarbonReadDataHolder[] values) {
+ this.values = values;
+ }
+
+ @Override public CarbonReadDataHolder[] getValues() {
+ return values;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/impl/DFSFileHolderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/DFSFileHolderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/DFSFileHolderImpl.java
new file mode 100644
index 0000000..443344f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/DFSFileHolderImpl.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.datastorage.impl;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastorage.FileHolder;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+
+public class DFSFileHolderImpl implements FileHolder {
+ /**
+ * cache to hold filename and its stream
+ */
+ private Map<String, FSDataInputStream> fileNameAndStreamCache;
+
+ public DFSFileHolderImpl() {
+ this.fileNameAndStreamCache =
+ new HashMap<String, FSDataInputStream>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ }
+
+ @Override public byte[] readByteArray(String filePath, long offset, int length)
+ throws IOException {
+ FSDataInputStream fileChannel = updateCache(filePath);
+ return read(fileChannel, length, offset);
+ }
+
+ /**
+ * This method will be used to check whether stream is already present in
+ * cache or not for filepath if not present then create it and then add to
+ * cache, other wise get from cache
+ *
+ * @param filePath fully qualified file path
+ * @return channel
+ */
+ private FSDataInputStream updateCache(String filePath) throws IOException {
+ FSDataInputStream fileChannel = fileNameAndStreamCache.get(filePath);
+ if (null == fileChannel) {
+ Path pt = new Path(filePath);
+ FileSystem fs = FileSystem.get(FileFactory.getConfiguration());
+ fileChannel = fs.open(pt);
+ fileNameAndStreamCache.put(filePath, fileChannel);
+ }
+ return fileChannel;
+ }
+
+ /**
+ * This method will be used to read from file based on number of bytes to be read and positon
+ *
+ * @param channel file channel
+ * @param size number of bytes
+ * @param offset position
+ * @return byte buffer
+ */
+ private byte[] read(FSDataInputStream channel, int size, long offset) throws IOException {
+ byte[] byteBffer = new byte[size];
+ channel.seek(offset);
+ channel.readFully(byteBffer);
+ return byteBffer;
+ }
+
+ /**
+ * This method will be used to read from file based on number of bytes to be read and positon
+ *
+ * @param channel file channel
+ * @param size number of bytes
+ * @return byte buffer
+ */
+ private byte[] read(FSDataInputStream channel, int size) throws IOException {
+ byte[] byteBffer = new byte[size];
+ channel.readFully(byteBffer);
+ return byteBffer;
+ }
+
+ @Override public int readInt(String filePath, long offset) throws IOException {
+ FSDataInputStream fileChannel = updateCache(filePath);
+ fileChannel.seek(offset);
+ return fileChannel.readInt();
+ }
+
+ @Override public long readDouble(String filePath, long offset) throws IOException {
+ FSDataInputStream fileChannel = updateCache(filePath);
+ fileChannel.seek(offset);
+ return fileChannel.readLong();
+ }
+
+ @Override public void finish() throws IOException {
+ for (Entry<String, FSDataInputStream> entry : fileNameAndStreamCache.entrySet()) {
+ FSDataInputStream channel = entry.getValue();
+ if (null != channel) {
+ channel.close();
+ }
+ }
+ }
+
+ @Override public byte[] readByteArray(String filePath, int length) throws IOException {
+ FSDataInputStream fileChannel = updateCache(filePath);
+ return read(fileChannel, length);
+ }
+
+ @Override public long readLong(String filePath, long offset) throws IOException {
+ FSDataInputStream fileChannel = updateCache(filePath);
+ fileChannel.seek(offset);
+ return fileChannel.readLong();
+ }
+
+ @Override public int readInt(String filePath) throws IOException {
+ FSDataInputStream fileChannel = updateCache(filePath);
+ return fileChannel.readInt();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/impl/FileFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/FileFactory.java b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/FileFactory.java
new file mode 100644
index 0000000..ca9956d
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/FileFactory.java
@@ -0,0 +1,485 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.impl;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.zip.GZIPInputStream;
+
+import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.datastorage.filesystem.AlluxioCarbonFile;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastorage.filesystem.HDFSCarbonFile;
+import org.apache.carbondata.core.datastorage.filesystem.LocalCarbonFile;
+import org.apache.carbondata.core.datastorage.filesystem.ViewFSCarbonFile;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.compress.BZip2Codec;
+import org.apache.hadoop.io.compress.GzipCodec;
+
+public final class FileFactory {
+ private static Configuration configuration = null;
+
+ static {
+ configuration = new Configuration();
+ configuration.addResource(new Path("../core-default.xml"));
+ }
+
+ private FileFactory() {
+
+ }
+
+ public static Configuration getConfiguration() {
+ return configuration;
+ }
+
+ public static FileHolder getFileHolder(FileType fileType) {
+ switch (fileType) {
+ case LOCAL:
+ return new FileHolderImpl();
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ return new DFSFileHolderImpl();
+ default:
+ return new FileHolderImpl();
+ }
+ }
+
+ public static FileType getFileType(String path) {
+ if (path.startsWith(CarbonUtil.HDFS_PREFIX)) {
+ return FileType.HDFS;
+ }
+ else if (path.startsWith(CarbonUtil.ALLUXIO_PREFIX)) {
+ return FileType.ALLUXIO;
+ }
+ else if (path.startsWith(CarbonUtil.VIEWFS_PREFIX)) {
+ return FileType.VIEWFS;
+ }
+ return FileType.LOCAL;
+ }
+
+ public static CarbonFile getCarbonFile(String path, FileType fileType) {
+ switch (fileType) {
+ case LOCAL:
+ return new LocalCarbonFile(path);
+ case HDFS:
+ return new HDFSCarbonFile(path);
+ case ALLUXIO:
+ return new AlluxioCarbonFile(path);
+ case VIEWFS:
+ return new ViewFSCarbonFile(path);
+ default:
+ return new LocalCarbonFile(path);
+ }
+ }
+
+ public static DataInputStream getDataInputStream(String path, FileType fileType)
+ throws IOException {
+ return getDataInputStream(path, fileType, -1);
+ }
+
+ public static DataInputStream getDataInputStream(String path, FileType fileType, int bufferSize)
+ throws IOException {
+ path = path.replace("\\", "/");
+ boolean gzip = path.endsWith(".gz");
+ boolean bzip2 = path.endsWith(".bz2");
+ InputStream stream;
+ switch (fileType) {
+ case LOCAL:
+ if (gzip) {
+ stream = new GZIPInputStream(new FileInputStream(path));
+ } else if (bzip2) {
+ stream = new BZip2CompressorInputStream(new FileInputStream(path));
+ } else {
+ stream = new FileInputStream(path);
+ }
+ break;
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ Path pt = new Path(path);
+ FileSystem fs = pt.getFileSystem(configuration);
+ if (bufferSize == -1) {
+ stream = fs.open(pt);
+ } else {
+ stream = fs.open(pt, bufferSize);
+ }
+ if (gzip) {
+ GzipCodec codec = new GzipCodec();
+ stream = codec.createInputStream(stream);
+ } else if (bzip2) {
+ BZip2Codec codec = new BZip2Codec();
+ stream = codec.createInputStream(stream);
+ }
+ break;
+ default:
+ throw new UnsupportedOperationException("unsupported file system");
+ }
+ return new DataInputStream(new BufferedInputStream(stream));
+ }
+
+ /**
+ * return the datainputStream which is seek to the offset of file
+ *
+ * @param path
+ * @param fileType
+ * @param bufferSize
+ * @param offset
+ * @return DataInputStream
+ * @throws IOException
+ */
+ public static DataInputStream getDataInputStream(String path, FileType fileType, int bufferSize,
+ long offset) throws IOException {
+ path = path.replace("\\", "/");
+ switch (fileType) {
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ Path pt = new Path(path);
+ FileSystem fs = pt.getFileSystem(configuration);
+ FSDataInputStream stream = fs.open(pt, bufferSize);
+ stream.seek(offset);
+ return new DataInputStream(new BufferedInputStream(stream));
+ default:
+ FileInputStream fis = new FileInputStream(path);
+ long actualSkipSize = 0;
+ long skipSize = offset;
+ while (actualSkipSize != offset) {
+ actualSkipSize += fis.skip(skipSize);
+ skipSize = skipSize - actualSkipSize;
+ }
+ return new DataInputStream(new BufferedInputStream(fis));
+ }
+ }
+
+ public static DataOutputStream getDataOutputStream(String path, FileType fileType)
+ throws IOException {
+ path = path.replace("\\", "/");
+ switch (fileType) {
+ case LOCAL:
+ return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path)));
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ Path pt = new Path(path);
+ FileSystem fs = pt.getFileSystem(configuration);
+ FSDataOutputStream stream = fs.create(pt, true);
+ return stream;
+ default:
+ return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path)));
+ }
+ }
+
+ public static DataOutputStream getDataOutputStream(String path, FileType fileType, int bufferSize,
+ boolean append) throws IOException {
+ path = path.replace("\\", "/");
+ switch (fileType) {
+ case LOCAL:
+ return new DataOutputStream(
+ new BufferedOutputStream(new FileOutputStream(path, append), bufferSize));
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ Path pt = new Path(path);
+ FileSystem fs = pt.getFileSystem(configuration);
+ FSDataOutputStream stream = null;
+ if (append) {
+ // append to a file only if file already exists else file not found
+ // exception will be thrown by hdfs
+ if (CarbonUtil.isFileExists(path)) {
+ stream = fs.append(pt, bufferSize);
+ } else {
+ stream = fs.create(pt, true, bufferSize);
+ }
+ } else {
+ stream = fs.create(pt, true, bufferSize);
+ }
+ return stream;
+ default:
+ return new DataOutputStream(
+ new BufferedOutputStream(new FileOutputStream(path), bufferSize));
+ }
+ }
+
+ public static DataOutputStream getDataOutputStream(String path, FileType fileType, int bufferSize,
+ long blockSize) throws IOException {
+ path = path.replace("\\", "/");
+ switch (fileType) {
+ case LOCAL:
+ return new DataOutputStream(
+ new BufferedOutputStream(new FileOutputStream(path), bufferSize));
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ Path pt = new Path(path);
+ FileSystem fs = pt.getFileSystem(configuration);
+ FSDataOutputStream stream =
+ fs.create(pt, true, bufferSize, fs.getDefaultReplication(pt), blockSize);
+ return stream;
+ default:
+ return new DataOutputStream(
+ new BufferedOutputStream(new FileOutputStream(path), bufferSize));
+ }
+ }
+
+ /**
+ * This method checks the given path exists or not and also is it file or
+ * not if the performFileCheck is true
+ *
+ * @param filePath - Path
+ * @param fileType - FileType Local/HDFS
+ * @param performFileCheck - Provide false for folders, true for files and
+ */
+ public static boolean isFileExist(String filePath, FileType fileType, boolean performFileCheck)
+ throws IOException {
+ filePath = filePath.replace("\\", "/");
+ switch (fileType) {
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ Path path = new Path(filePath);
+ FileSystem fs = path.getFileSystem(configuration);
+ if (performFileCheck) {
+ return fs.exists(path) && fs.isFile(path);
+ } else {
+ return fs.exists(path);
+ }
+
+ case LOCAL:
+ default:
+ File defaultFile = new File(filePath);
+
+ if (performFileCheck) {
+ return defaultFile.exists() && defaultFile.isFile();
+ } else {
+ return defaultFile.exists();
+ }
+ }
+ }
+
+ /**
+ * This method checks the given path exists or not and also is it file or
+ * not if the performFileCheck is true
+ *
+ * @param filePath - Path
+ * @param fileType - FileType Local/HDFS
+ */
+ public static boolean isFileExist(String filePath, FileType fileType) throws IOException {
+ filePath = filePath.replace("\\", "/");
+ switch (fileType) {
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ Path path = new Path(filePath);
+ FileSystem fs = path.getFileSystem(configuration);
+ return fs.exists(path);
+
+ case LOCAL:
+ default:
+ File defaultFile = new File(filePath);
+ return defaultFile.exists();
+ }
+ }
+
+ public static boolean createNewFile(String filePath, FileType fileType) throws IOException {
+ filePath = filePath.replace("\\", "/");
+ switch (fileType) {
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ Path path = new Path(filePath);
+ FileSystem fs = path.getFileSystem(configuration);
+ return fs.createNewFile(path);
+
+ case LOCAL:
+ default:
+ File file = new File(filePath);
+ return file.createNewFile();
+ }
+ }
+
+ public static boolean deleteFile(String filePath, FileType fileType) throws IOException {
+ filePath = filePath.replace("\\", "/");
+ switch (fileType) {
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ Path path = new Path(filePath);
+ FileSystem fs = path.getFileSystem(configuration);
+ return fs.delete(path, true);
+
+ case LOCAL:
+ default:
+ File file = new File(filePath);
+ return deleteAllFilesOfDir(file);
+ }
+ }
+
+ public static boolean deleteAllFilesOfDir(File path) {
+ if (!path.exists()) {
+ return true;
+ }
+ if (path.isFile()) {
+ return path.delete();
+ }
+ File[] files = path.listFiles();
+ for (int i = 0; i < files.length; i++) {
+ deleteAllFilesOfDir(files[i]);
+ }
+ return path.delete();
+ }
+
+ public static boolean mkdirs(String filePath, FileType fileType) throws IOException {
+ filePath = filePath.replace("\\", "/");
+ switch (fileType) {
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ Path path = new Path(filePath);
+ FileSystem fs = path.getFileSystem(configuration);
+ return fs.mkdirs(path);
+ case LOCAL:
+ default:
+ File file = new File(filePath);
+ return file.mkdirs();
+ }
+ }
+
+ /**
+ * for getting the dataoutput stream using the hdfs filesystem append API.
+ *
+ * @param path
+ * @param fileType
+ * @return
+ * @throws IOException
+ */
+ public static DataOutputStream getDataOutputStreamUsingAppend(String path, FileType fileType)
+ throws IOException {
+ path = path.replace("\\", "/");
+ switch (fileType) {
+ case LOCAL:
+ return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path, true)));
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ Path pt = new Path(path);
+ FileSystem fs = pt.getFileSystem(configuration);
+ FSDataOutputStream stream = fs.append(pt);
+ return stream;
+ default:
+ return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path)));
+ }
+ }
+
+ /**
+ * for creating a new Lock file and if it is successfully created
+ * then in case of abrupt shutdown then the stream to that file will be closed.
+ *
+ * @param filePath
+ * @param fileType
+ * @return
+ * @throws IOException
+ */
+ public static boolean createNewLockFile(String filePath, FileType fileType) throws IOException {
+ filePath = filePath.replace("\\", "/");
+ switch (fileType) {
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ Path path = new Path(filePath);
+ FileSystem fs = path.getFileSystem(configuration);
+ if (fs.createNewFile(path)) {
+ fs.deleteOnExit(path);
+ return true;
+ }
+ return false;
+ case LOCAL:
+ default:
+ File file = new File(filePath);
+ return file.createNewFile();
+ }
+ }
+
+ public enum FileType {
+ LOCAL, HDFS, ALLUXIO, VIEWFS
+ }
+
+ /**
+ * below method will be used to update the file path
+ * for local type
+ * it removes the file:/ from the path
+ *
+ * @param filePath
+ * @return updated file path without url for local
+ */
+ public static String getUpdatedFilePath(String filePath) {
+ FileType fileType = getFileType(filePath);
+ switch (fileType) {
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ return filePath;
+ case LOCAL:
+ default:
+ Path pathWithoutSchemeAndAuthority =
+ Path.getPathWithoutSchemeAndAuthority(new Path(filePath));
+ return pathWithoutSchemeAndAuthority.toString();
+ }
+ }
+
+ /**
+ * It computes size of directory
+ *
+ * @param filePath
+ * @return size in bytes
+ * @throws IOException
+ */
+ public static long getDirectorySize(String filePath) throws IOException {
+ FileType fileType = getFileType(filePath);
+ switch (fileType) {
+ case HDFS:
+ case ALLUXIO:
+ case VIEWFS:
+ Path path = new Path(filePath);
+ FileSystem fs = path.getFileSystem(configuration);
+ return fs.getContentSummary(path).getLength();
+ case LOCAL:
+ default:
+ File file = new File(filePath);
+ return FileUtils.sizeOfDirectory(file);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/impl/FileHolderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/FileHolderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/FileHolderImpl.java
new file mode 100644
index 0000000..843ec5a
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/FileHolderImpl.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.impl;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastorage.FileHolder;
+
+public class FileHolderImpl implements FileHolder {
+ /**
+ * cache to hold filename and its stream
+ */
+ private Map<String, FileChannel> fileNameAndStreamCache;
+
+ /**
+ * FileHolderImpl Constructor
+ * It will create the cache
+ */
+ public FileHolderImpl() {
+ this.fileNameAndStreamCache =
+ new HashMap<String, FileChannel>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ }
+
+ public FileHolderImpl(int capacity) {
+ this.fileNameAndStreamCache = new HashMap<String, FileChannel>(capacity);
+ }
+
+ /**
+ * This method will be used to read the byte array from file based on offset
+ * and length(number of bytes) need to read
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @param length number of bytes to be read
+ * @return read byte array
+ */
+ @Override public byte[] readByteArray(String filePath, long offset, int length)
+ throws IOException {
+ FileChannel fileChannel = updateCache(filePath);
+ ByteBuffer byteBffer = read(fileChannel, length, offset);
+ return byteBffer.array();
+ }
+
+ /**
+ * This method will be used to close all the streams currently present in the cache
+ */
+ @Override public void finish() throws IOException {
+ for (Entry<String, FileChannel> entry : fileNameAndStreamCache.entrySet()) {
+ FileChannel channel = entry.getValue();
+ if (null != channel) {
+ channel.close();
+ }
+ }
+ }
+
+ /**
+ * This method will be used to read int from file from postion(offset), here
+ * length will be always 4 bacause int byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read int
+ */
+ @Override public int readInt(String filePath, long offset) throws IOException {
+ FileChannel fileChannel = updateCache(filePath);
+ ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.INT_SIZE_IN_BYTE, offset);
+ return byteBffer.getInt();
+ }
+
+ /**
+ * This method will be used to read int from file from postion(offset), here
+ * length will be always 4 bacause int byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @return read int
+ */
+ @Override public int readInt(String filePath) throws IOException {
+ FileChannel fileChannel = updateCache(filePath);
+ ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.INT_SIZE_IN_BYTE);
+ return byteBffer.getInt();
+ }
+
+ /**
+ * This method will be used to read int from file from postion(offset), here
+ * length will be always 4 bacause int byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read int
+ */
+ @Override public long readDouble(String filePath, long offset) throws IOException {
+ FileChannel fileChannel = updateCache(filePath);
+ ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.LONG_SIZE_IN_BYTE, offset);
+ return byteBffer.getLong();
+ }
+
+ /**
+ * This method will be used to check whether stream is already present in
+ * cache or not for filepath if not present then create it and then add to
+ * cache, other wise get from cache
+ *
+ * @param filePath fully qualified file path
+ * @return channel
+ */
+ private FileChannel updateCache(String filePath) throws FileNotFoundException {
+ FileChannel fileChannel = fileNameAndStreamCache.get(filePath);
+ if (null == fileChannel) {
+ FileInputStream stream = new FileInputStream(filePath);
+ fileChannel = stream.getChannel();
+ fileNameAndStreamCache.put(filePath, fileChannel);
+ }
+ return fileChannel;
+ }
+
+ /**
+ * This method will be used to read from file based on number of bytes to be read and positon
+ *
+ * @param channel file channel
+ * @param size number of bytes
+ * @param offset position
+ * @return byte buffer
+ */
+ private ByteBuffer read(FileChannel channel, int size, long offset) throws IOException {
+ ByteBuffer byteBffer = ByteBuffer.allocate(size);
+ channel.position(offset);
+ channel.read(byteBffer);
+ byteBffer.rewind();
+ return byteBffer;
+ }
+
+ /**
+ * This method will be used to read from file based on number of bytes to be read and positon
+ *
+ * @param channel file channel
+ * @param size number of bytes
+ * @return byte buffer
+ */
+ private ByteBuffer read(FileChannel channel, int size) throws IOException {
+ ByteBuffer byteBffer = ByteBuffer.allocate(size);
+ channel.read(byteBffer);
+ byteBffer.rewind();
+ return byteBffer;
+ }
+
+
+ /**
+ * This method will be used to read the byte array from file based on length(number of bytes)
+ *
+ * @param filePath fully qualified file path
+ * @param length number of bytes to be read
+ * @return read byte array
+ */
+ @Override public byte[] readByteArray(String filePath, int length) throws IOException {
+ FileChannel fileChannel = updateCache(filePath);
+ ByteBuffer byteBffer = read(fileChannel, length);
+ return byteBffer.array();
+ }
+
+ /**
+ * This method will be used to read long from file from postion(offset), here
+ * length will be always 8 bacause int byte size is 8
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read long
+ */
+ @Override public long readLong(String filePath, long offset) throws IOException {
+ FileChannel fileChannel = updateCache(filePath);
+ ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.LONG_SIZE_IN_BYTE, offset);
+ return byteBffer.getLong();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
new file mode 100644
index 0000000..8573187
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.impl.data.compressed;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastorage.NodeMeasureDataStore;
+import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+
+public abstract class AbstractHeavyCompressedDoubleArrayDataStore
+ implements NodeMeasureDataStore //NodeMeasureDataStore<double[]>
+{
+
+ private LogService LOGGER =
+ LogServiceFactory.getLogService(AbstractHeavyCompressedDoubleArrayDataStore.class.getName());
+
+ /**
+ * values.
+ */
+ protected ValueCompressionHolder[] values;
+
+ /**
+ * compressionModel.
+ */
+ protected WriterCompressModel compressionModel;
+
+ /**
+ * type
+ */
+ private char[] type;
+
+ /**
+ * AbstractHeavyCompressedDoubleArrayDataStore constructor.
+ *
+ * @param compressionModel
+ */
+ public AbstractHeavyCompressedDoubleArrayDataStore(WriterCompressModel compressionModel) {
+ this.compressionModel = compressionModel;
+ if (null != compressionModel) {
+ this.type = compressionModel.getType();
+ values =
+ new ValueCompressionHolder[compressionModel.getValueCompressionHolder().length];
+ }
+ }
+
+ // this method first invokes encoding routine to encode the data chunk,
+ // followed by invoking compression routine for preparing the data chunk for writing.
+ @Override public byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] dataHolder) {
+ byte[][] returnValue = new byte[values.length][];
+ for (int i = 0; i < compressionModel.getValueCompressionHolder().length; i++) {
+ values[i] = compressionModel.getValueCompressionHolder()[i];
+ if (type[i] != CarbonCommonConstants.BYTE_VALUE_MEASURE) {
+ // first perform encoding of the data chunk
+ values[i].setValue(
+ ValueCompressionUtil.getValueCompressor(compressionModel.getCompressionFinders()[i])
+ .getCompressedValues(compressionModel.getCompressionFinders()[i], dataHolder[i],
+ compressionModel.getMaxValue()[i],
+ compressionModel.getMantissa()[i]));
+ } else {
+ values[i].setValue(dataHolder[i].getWritableByteArrayValues());
+ }
+ values[i].compress();
+ returnValue[i] = values[i].getCompressedData();
+ }
+
+ return returnValue;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
new file mode 100644
index 0000000..ea42beb
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.impl.data.compressed;
+
+import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
+
+public class HeavyCompressedDoubleArrayDataInMemoryStore
+ extends AbstractHeavyCompressedDoubleArrayDataStore {
+
+ public HeavyCompressedDoubleArrayDataInMemoryStore(WriterCompressModel compressionModel) {
+ super(compressionModel);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/FileHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/FileHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/FileHolder.java
deleted file mode 100644
index 1f99158..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/FileHolder.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store;
-
-import java.io.IOException;
-
-public interface FileHolder {
- /**
- * This method will be used to read the byte array from file based on offset
- * and length(number of bytes) need to read
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @param length number of bytes to be read
- * @return read byte array
- */
- byte[] readByteArray(String filePath, long offset, int length) throws IOException;
-
- /**
- * This method will be used to read the byte array from file based on length(number of bytes)
- *
- * @param filePath fully qualified file path
- * @param length number of bytes to be read
- * @return read byte array
- */
- byte[] readByteArray(String filePath, int length) throws IOException;
-
- /**
- * This method will be used to read int from file from postion(offset), here
- * length will be always 4 bacause int byte size if 4
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @return read int
- */
- int readInt(String filePath, long offset) throws IOException;
-
- /**
- * This method will be used to read long from file from postion(offset), here
- * length will be always 8 bacause int byte size is 8
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @return read long
- */
- long readLong(String filePath, long offset) throws IOException;
-
- /**
- * This method will be used to read int from file from postion(offset), here
- * length will be always 4 bacause int byte size if 4
- *
- * @param filePath fully qualified file path
- * @return read int
- */
- int readInt(String filePath) throws IOException;
-
- /**
- * This method will be used to read long value from file from postion(offset), here
- * length will be always 8 because long byte size if 4
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @return read long
- */
- long readDouble(String filePath, long offset) throws IOException;
-
- /**
- * This method will be used to close all the streams currently present in the cache
- */
- void finish() throws IOException;
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/MeasureDataWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/MeasureDataWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/MeasureDataWrapper.java
deleted file mode 100644
index 80a4374..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/MeasureDataWrapper.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store;
-
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
-
-/**
- * MeasureDataWrapper, interface.
- */
-public interface MeasureDataWrapper {
- CarbonReadDataHolder[] getValues();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java
deleted file mode 100644
index e40dadd..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store;
-
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder;
-
-public interface NodeMeasureDataStore {
- /**
- * This method will be used to get the writable key array.
- * writable measure data array will hold below information:
- * <size of measure data array><measure data array>
- * total length will be 4 bytes for size + measure data array length
- *
- * @return writable array (compressed or normal)
- */
- byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] dataHolderArray);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForInt.java
deleted file mode 100644
index 013d873..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForInt.java
+++ /dev/null
@@ -1,226 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.columnar;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.util.ByteUtil;
-
-public class BlockIndexerStorageForInt implements IndexStorage<int[]> {
- private boolean alreadySorted;
-
- private int[] dataAfterComp;
-
- private int[] indexMap;
-
- private byte[][] keyBlock;
-
- private int[] dataIndexMap;
-
- private int totalSize;
-
- public BlockIndexerStorageForInt(byte[][] keyBlock, boolean compressData, boolean isNoDictionary,
- boolean isSortRequired) {
- ColumnWithIntIndex[] columnWithIndexs = createColumnWithIndexArray(keyBlock, isNoDictionary);
- if (isSortRequired) {
- Arrays.sort(columnWithIndexs);
- }
- compressMyOwnWay(extractDataAndReturnIndexes(columnWithIndexs, keyBlock));
- if (compressData) {
- compressDataMyOwnWay(columnWithIndexs);
- }
- }
-
- /**
- * Create an object with each column array and respective index
- *
- * @return
- */
- private ColumnWithIntIndex[] createColumnWithIndexArray(byte[][] keyBlock,
- boolean isNoDictionary) {
- ColumnWithIntIndex[] columnWithIndexs;
- if (isNoDictionary) {
- columnWithIndexs = new ColumnWithIntIndexForHighCard[keyBlock.length];
- for (int i = 0; i < columnWithIndexs.length; i++) {
- columnWithIndexs[i] = new ColumnWithIntIndexForHighCard(keyBlock[i], i);
- }
-
- } else {
- columnWithIndexs = new ColumnWithIntIndex[keyBlock.length];
- for (int i = 0; i < columnWithIndexs.length; i++) {
- columnWithIndexs[i] = new ColumnWithIntIndex(keyBlock[i], i);
- }
- }
-
- return columnWithIndexs;
- }
-
- private int[] extractDataAndReturnIndexes(ColumnWithIntIndex[] columnWithIndexs,
- byte[][] keyBlock) {
- int[] indexes = new int[columnWithIndexs.length];
- for (int i = 0; i < indexes.length; i++) {
- indexes[i] = columnWithIndexs[i].getIndex();
- keyBlock[i] = columnWithIndexs[i].getColumn();
- }
- this.keyBlock = keyBlock;
- return indexes;
- }
-
- /**
- * It compresses depends up on the sequence numbers.
- * [1,2,3,4,6,8,10,11,12,13] is translated to [1,4,6,8,10,13] and [0,6]. In
- * first array the start and end of sequential numbers and second array
- * keeps the indexes of where sequential numbers starts. If there is no
- * sequential numbers then the same array it returns with empty second
- * array.
- *
- * @param indexes
- */
- public void compressMyOwnWay(int[] indexes) {
- List<Integer> list = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
- List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
- int k = 0;
- int i = 1;
- for (; i < indexes.length; i++) {
- if (indexes[i] - indexes[i - 1] == 1) {
- k++;
- } else {
- if (k > 0) {
- map.add((list.size()));
- list.add(indexes[i - k - 1]);
- list.add(indexes[i - 1]);
- } else {
- list.add(indexes[i - 1]);
- }
- k = 0;
- }
- }
- if (k > 0) {
- map.add((list.size()));
- list.add(indexes[i - k - 1]);
- list.add(indexes[i - 1]);
- } else {
- list.add(indexes[i - 1]);
- }
- dataAfterComp = convertToArray(list);
- if (indexes.length == dataAfterComp.length) {
- indexMap = new int[0];
- } else {
- indexMap = convertToArray(map);
- }
- if (dataAfterComp.length == 2 && indexMap.length == 1) {
- alreadySorted = true;
- }
- }
-
- private int[] convertToArray(List<Integer> list) {
- int[] shortArray = new int[list.size()];
- for (int i = 0; i < shortArray.length; i++) {
- shortArray[i] = list.get(i);
- }
- return shortArray;
- }
-
- /**
- * @return the alreadySorted
- */
- public boolean isAlreadySorted() {
- return alreadySorted;
- }
-
- /**
- * @return the dataAfterComp
- */
- public int[] getDataAfterComp() {
- return dataAfterComp;
- }
-
- /**
- * @return the indexMap
- */
- public int[] getIndexMap() {
- return indexMap;
- }
-
- /**
- * @return the keyBlock
- */
- public byte[][] getKeyBlock() {
- return keyBlock;
- }
-
- private void compressDataMyOwnWay(ColumnWithIntIndex[] indexes) {
- byte[] prvKey = indexes[0].getColumn();
- List<ColumnWithIntIndex> list =
- new ArrayList<ColumnWithIntIndex>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
- list.add(indexes[0]);
- int counter = 1;
- int start = 0;
- List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
- for (int i = 1; i < indexes.length; i++) {
- if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, indexes[i].getColumn()) != 0) {
- prvKey = indexes[i].getColumn();
- list.add(indexes[i]);
- map.add(start);
- map.add(counter);
- start += counter;
- counter = 1;
- continue;
- }
- counter++;
- }
- map.add(start);
- map.add(counter);
- this.keyBlock = convertToKeyArray(list);
- if (indexes.length == keyBlock.length) {
- dataIndexMap = new int[0];
- } else {
- dataIndexMap = convertToArray(map);
- }
- }
-
- private byte[][] convertToKeyArray(List<ColumnWithIntIndex> list) {
- byte[][] shortArray = new byte[list.size()][];
- for (int i = 0; i < shortArray.length; i++) {
- shortArray[i] = list.get(i).getColumn();
- totalSize += shortArray[i].length;
- }
- return shortArray;
- }
-
- @Override public int[] getDataIndexMap() {
- return dataIndexMap;
- }
-
- @Override public int getTotalSize() {
- return totalSize;
- }
-
- @Override public byte[] getMin() {
- return keyBlock[0];
- }
-
- @Override public byte[] getMax() {
- return keyBlock[keyBlock.length - 1];
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForNoInvertedIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForNoInvertedIndex.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForNoInvertedIndex.java
deleted file mode 100644
index c7d43cf..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForNoInvertedIndex.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.columnar;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.List;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.util.ByteUtil;
-
-public class BlockIndexerStorageForNoInvertedIndex implements IndexStorage<int[]> {
- private byte[][] keyBlock;
- private byte[][] sortedBlock;
- private int totalSize;
- private int[] dataIndexMap;
-
- public BlockIndexerStorageForNoInvertedIndex(byte[][] keyBlockInput, boolean compressData,
- boolean isNoDictionary) {
- // without invertedindex but can be RLE
- if (compressData) {
- // with RLE
- byte[] prvKey = keyBlockInput[0];
- List<byte[]> list = new ArrayList<byte[]>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
- list.add(keyBlockInput[0]);
- int counter = 1;
- int start = 0;
- List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
- int length = keyBlockInput.length;
- for(int i = 1; i < length; i++) {
- if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, keyBlockInput[i]) != 0) {
- prvKey = keyBlockInput[i];
- list.add(keyBlockInput[i]);
- map.add(start);
- map.add(counter);
- start += counter;
- counter = 1;
- continue;
- }
- counter++;
- }
- map.add(start);
- map.add(counter);
- this.keyBlock = convertToKeyArray(list);
- if (keyBlockInput.length == this.keyBlock.length) {
- dataIndexMap = new int[0];
- } else {
- dataIndexMap = convertToArray(map);
- }
- } else {
- this.keyBlock = keyBlockInput;
- dataIndexMap = new int[0];
- }
-
- this.sortedBlock = new byte[keyBlock.length][];
- System.arraycopy(keyBlock, 0, sortedBlock, 0, keyBlock.length);
- if (isNoDictionary) {
- Arrays.sort(sortedBlock, new Comparator<byte[]>() {
- @Override
- public int compare(byte[] col1, byte[] col2) {
- return ByteUtil.UnsafeComparer.INSTANCE
- .compareTo(col1, 2, col1.length - 2, col2, 2, col2.length - 2);
- }
- });
- } else {
- Arrays.sort(sortedBlock, new Comparator<byte[]>() {
- @Override
- public int compare(byte[] col1, byte[] col2) {
- return ByteUtil.UnsafeComparer.INSTANCE.compareTo(col1, col2);
- }
- });
- }
-
- }
-
- private int[] convertToArray(List<Integer> list) {
- int[] shortArray = new int[list.size()];
- for(int i = 0; i < shortArray.length; i++) {
- shortArray[i] = list.get(i);
- }
- return shortArray;
- }
-
- private byte[][] convertToKeyArray(List<byte[]> list) {
- byte[][] shortArray = new byte[list.size()][];
- for (int i = 0; i < shortArray.length; i++) {
- shortArray[i] = list.get(i);
- totalSize += shortArray[i].length;
- }
- return shortArray;
- }
-
- @Override
- public int[] getDataIndexMap() {
- return dataIndexMap;
- }
-
- @Override
- public int getTotalSize() {
- return totalSize;
- }
-
- @Override
- public boolean isAlreadySorted() {
- return true;
- }
-
- /**
- * no use
- * @return
- */
- @Override
- public int[] getDataAfterComp() {
- return new int[0];
- }
-
- /**
- * no use
- * @return
- */
- @Override
- public int[] getIndexMap() {
- return new int[0];
- }
-
- /**
- * @return the keyBlock
- */
- public byte[][] getKeyBlock() {
- return keyBlock;
- }
-
- @Override public byte[] getMin() {
- return sortedBlock[0];
- }
-
- @Override public byte[] getMax() {
- return sortedBlock[sortedBlock.length - 1];
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java
deleted file mode 100644
index cf9ba40..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.datastorage.store.columnar;
-
-public class ColumnGroupModel {
-
- /**
- * number of columns in columnar block
- */
- private int[] columnSplit;
-
- /**
- * total number of columns
- */
- private int noOfColumnsStore;
-
- /**
- * whether given index is columnar or not
- * true: columnar
- * false: row block
- */
- private boolean[] columnarStore;
-
- /**
- * column groups
- * e.g
- * {{0,1,2},3,4,{5,6}}
- */
- private int[][] columnGroups;
-
- /**
- * return columnSplit
- *
- * @return
- */
- public int[] getColumnSplit() {
- return columnSplit;
- }
-
- /**
- * set columnSplit
- *
- * @param split
- */
- public void setColumnSplit(int[] split) {
- this.columnSplit = split;
- }
-
- /**
- * @return no of columnar block
- */
- public int getNoOfColumnStore() {
- return this.noOfColumnsStore;
- }
-
- /**
- * set no of columnar block
- *
- * @param noOfColumnsStore
- */
- public void setNoOfColumnStore(int noOfColumnsStore) {
- this.noOfColumnsStore = noOfColumnsStore;
- }
-
- /**
- * it's an identifier for row block or single column block
- *
- * @param columnarStore
- */
- public void setColumnarStore(boolean[] columnarStore) {
- this.columnarStore = columnarStore;
- }
-
- /**
- * set column groups
- *
- * @param columnGroups
- */
- public void setColumnGroup(int[][] columnGroups) {
- this.columnGroups = columnGroups;
- }
-
- /**
- * check if given column group is columnar
- *
- * @param colGroup
- * @return true if given block is columnar
- */
- public boolean isColumnar(int colGroup) {
- return columnarStore[colGroup];
- }
-
- /**
- * @return columngroups
- */
- public int[][] getColumnGroup() {
- return this.columnGroups;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndex.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndex.java
deleted file mode 100644
index 36606a5..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndex.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.columnar;
-
-import java.util.Arrays;
-
-import org.apache.carbondata.core.util.ByteUtil;
-
-public class ColumnWithIntIndex implements Comparable<ColumnWithIntIndex> {
- protected byte[] column;
-
- private int index;
-
- public ColumnWithIntIndex(byte[] column, int index) {
- this.column = column;
- this.index = index;
- }
-
- public ColumnWithIntIndex() {
- }
-
- /**
- * @return the column
- */
- public byte[] getColumn() {
- return column;
- }
-
- /**
- * @param column the column to set
- */
- public void setColumn(byte[] column) {
- this.column = column;
- }
-
- /**
- * @return the index
- */
- public int getIndex() {
- return index;
- }
-
- /**
- * @param index the index to set
- */
- public void setIndex(int index) {
- this.index = index;
- }
-
- @Override public int compareTo(ColumnWithIntIndex o) {
- return ByteUtil.UnsafeComparer.INSTANCE.compareTo(column, o.column);
- }
-
- @Override public boolean equals(Object obj) {
- if(obj == null || getClass() != obj.getClass()) {
- return false;
- }
- ColumnWithIntIndex o = (ColumnWithIntIndex)obj;
- return Arrays.equals(column, o.column) && index == o.index;
- }
-
- @Override public int hashCode() {
- return Arrays.hashCode(column) + index;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndexForHighCard.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndexForHighCard.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndexForHighCard.java
deleted file mode 100644
index 61a1165..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndexForHighCard.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.columnar;
-
-import java.util.Arrays;
-
-import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
-
-public class ColumnWithIntIndexForHighCard extends ColumnWithIntIndex
- implements Comparable<ColumnWithIntIndex> {
-
- public ColumnWithIntIndexForHighCard(byte[] column, int index) {
- super(column, index);
- }
-
- @Override public int compareTo(ColumnWithIntIndex o) {
- return UnsafeComparer.INSTANCE
- .compareTo(column, 2, column.length - 2, o.column, 2, o.column.length - 2);
- }
-
- @Override public boolean equals(Object obj) {
- if(obj == null || getClass() != obj.getClass()) {
- return false;
- }
- ColumnWithIntIndexForHighCard o = (ColumnWithIntIndexForHighCard)obj;
- return Arrays.equals(column, o.column) && getIndex() == o.getIndex();
- }
-
- @Override public int hashCode() {
- return Arrays.hashCode(column) + getIndex();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolder.java
deleted file mode 100644
index 29887a3..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolder.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.columnar;
-
-import java.nio.ByteBuffer;
-
-public class ColumnarKeyStoreDataHolder {
- private byte[] keyblockData;
- private ColumnarKeyStoreMetadata columnarKeyStoreMetadata;
-
- public ColumnarKeyStoreDataHolder(final byte[] keyblockData,
- final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) {
- this.keyblockData = keyblockData;
- this.columnarKeyStoreMetadata = columnarKeyStoreMetadata;
- }
-
- public ColumnarKeyStoreDataHolder(final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) {
- this.columnarKeyStoreMetadata = columnarKeyStoreMetadata;
- }
-
- public int getSurrogateKey(int columnIndex) {
- byte[] actual = new byte[4];
- int startIndex;
- if (null != columnarKeyStoreMetadata.getColumnReverseIndex()) {
- startIndex =
- columnarKeyStoreMetadata.getColumnReverseIndex()[columnIndex] * columnarKeyStoreMetadata
- .getEachRowSize();
- } else {
- startIndex = columnIndex * columnarKeyStoreMetadata.getEachRowSize();
- }
- int destPos = 4 - columnarKeyStoreMetadata.getEachRowSize();
- System.arraycopy(keyblockData, startIndex, actual, destPos,
- columnarKeyStoreMetadata.getEachRowSize());
- return ByteBuffer.wrap(actual).getInt();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreMetadata.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreMetadata.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreMetadata.java
deleted file mode 100644
index 7754ddb..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreMetadata.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.columnar;
-
-class ColumnarKeyStoreMetadata {
-
- private int[] columnReverseIndex;
-
- private int eachRowSize;
-
- ColumnarKeyStoreMetadata(int eachRowSize) {
- this.eachRowSize = eachRowSize;
- }
-
- /**
- * @return the eachRowSize
- */
- int getEachRowSize() {
- return eachRowSize;
- }
-
- /**
- * @return the columnReverseIndex
- */
- int[] getColumnReverseIndex() {
- return columnReverseIndex;
- }
-
- /**
- * @param columnReverseIndex the columnReverseIndex to set
- */
- void setColumnReverseIndex(int[] columnReverseIndex) {
- this.columnReverseIndex = columnReverseIndex;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/IndexStorage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/IndexStorage.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/IndexStorage.java
deleted file mode 100644
index e1f4548..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/IndexStorage.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.columnar;
-
-public interface IndexStorage<T> {
- boolean isAlreadySorted();
-
- T getDataAfterComp();
-
- T getIndexMap();
-
- byte[][] getKeyBlock();
-
- T getDataIndexMap();
-
- int getTotalSize();
-
- /**
- * @return min value of block
- */
- byte[] getMin();
-
- /**
- * @return max value of block
- */
- byte[] getMax();
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/UnBlockIndexer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/UnBlockIndexer.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/UnBlockIndexer.java
deleted file mode 100644
index 149facb..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/UnBlockIndexer.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.columnar;
-
-import java.util.Arrays;
-
-public final class UnBlockIndexer {
-
- private UnBlockIndexer() {
-
- }
-
- public static int[] uncompressIndex(int[] indexData, int[] indexMap) {
- int actualSize = indexData.length;
- for (int i = 0; i < indexMap.length; i++) {
- actualSize += indexData[indexMap[i] + 1] - indexData[indexMap[i]] - 1;
- }
- int[] indexes = new int[actualSize];
- int k = 0;
- for (int i = 0; i < indexData.length; i++) {
- int index = Arrays.binarySearch(indexMap, i);
- if (index > -1) {
- for (int j = indexData[indexMap[index]]; j <= indexData[indexMap[index] + 1]; j++) {
- indexes[k] = j;
- k++;
- }
- i++;
- } else {
- indexes[k] = indexData[i];
- k++;
- }
- }
- return indexes;
- }
-
- public static byte[] uncompressData(byte[] data, int[] index, int keyLen) {
- if (index.length < 1) {
- return data;
- }
- int numberOfCopy = 0;
- int actualSize = 0;
- int srcPos = 0;
- int destPos = 0;
- for (int i = 1; i < index.length; i += 2) {
- actualSize += index[i];
- }
- byte[] uncompressedData = new byte[actualSize * keyLen];
- int picIndex = 0;
- for (int i = 0; i < data.length; i += keyLen) {
- numberOfCopy = index[picIndex * 2 + 1];
- picIndex++;
- for (int j = 0; j < numberOfCopy; j++) {
- System.arraycopy(data, srcPos, uncompressedData, destPos, keyLen);
- destPos += keyLen;
- }
- srcPos += keyLen;
- }
- return uncompressedData;
- }
-
-}
[17/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/scan/executor/util/QueryUtil.java
deleted file mode 100644
index 4f0789f..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/executor/util/QueryUtil.java
+++ /dev/null
@@ -1,882 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.executor.util;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.TreeSet;
-
-import org.apache.carbondata.core.cache.Cache;
-import org.apache.carbondata.core.cache.CacheProvider;
-import org.apache.carbondata.core.cache.CacheType;
-import org.apache.carbondata.core.cache.dictionary.Dictionary;
-import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.metadata.CarbonMetadata;
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.core.metadata.Encoding;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.scan.complextypes.ArrayQueryType;
-import org.apache.carbondata.scan.complextypes.PrimitiveQueryType;
-import org.apache.carbondata.scan.complextypes.StructQueryType;
-import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.filter.GenericQueryType;
-import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.scan.model.QueryDimension;
-import org.apache.carbondata.scan.model.QueryMeasure;
-import org.apache.carbondata.scan.model.QueryModel;
-
-import org.apache.commons.lang3.ArrayUtils;
-
-/**
- * Utility class for query execution
- */
-public class QueryUtil {
-
- /**
- * Below method will be used to get the masked byte range based on the query
- * dimension. It will give the range in the mdkey. This will be used to get
- * the actual key array from masked mdkey
- *
- * @param queryDimensions query dimension selected in query
- * @param keyGenerator key generator
- * @return masked key
- */
- public static int[] getMaskedByteRange(List<QueryDimension> queryDimensions,
- KeyGenerator keyGenerator) {
- Set<Integer> byteRangeSet = new TreeSet<Integer>();
- int[] byteRange = null;
- for (int i = 0; i < queryDimensions.size(); i++) {
-
- // as no dictionary column and complex type columns
- // are not selected in the mdkey
- // so we will not select the those dimension for calculating the
- // range
- if (queryDimensions.get(i).getDimension().getKeyOrdinal() == -1) {
- continue;
- }
- // get the offset of the dimension in the mdkey
- byteRange =
- keyGenerator.getKeyByteOffsets(queryDimensions.get(i).getDimension().getKeyOrdinal());
- for (int j = byteRange[0]; j <= byteRange[1]; j++) {
- byteRangeSet.add(j);
- }
- }
- int[] maksedByteRange = new int[byteRangeSet.size()];
- int index = 0;
- Iterator<Integer> iterator = byteRangeSet.iterator();
- // add the masked byte range
- while (iterator.hasNext()) {
- maksedByteRange[index++] = iterator.next();
- }
- return maksedByteRange;
- }
-
- public static int[] getMaskedByteRangeBasedOrdinal(List<Integer> ordinals,
- KeyGenerator keyGenerator) {
- Set<Integer> byteRangeSet = new TreeSet<Integer>();
- int[] byteRange = null;
- for (int i = 0; i < ordinals.size(); i++) {
-
- // get the offset of the dimension in the mdkey
- byteRange = keyGenerator.getKeyByteOffsets(ordinals.get(i));
- for (int j = byteRange[0]; j <= byteRange[1]; j++) {
- byteRangeSet.add(j);
- }
- }
- int[] maksedByteRange = new int[byteRangeSet.size()];
- int index = 0;
- Iterator<Integer> iterator = byteRangeSet.iterator();
- // add the masked byte range
- while (iterator.hasNext()) {
- maksedByteRange[index++] = iterator.next();
- }
- return maksedByteRange;
- }
-
- /**
- * Below method will return the max key based on the dimension ordinal
- *
- * @param keyOrdinalList
- * @param generator
- * @return
- * @throws KeyGenException
- */
- public static byte[] getMaxKeyBasedOnOrinal(List<Integer> keyOrdinalList, KeyGenerator generator)
- throws KeyGenException {
- long[] max = new long[generator.getDimCount()];
- Arrays.fill(max, 0L);
-
- for (int i = 0; i < keyOrdinalList.size(); i++) {
- // adding for dimension which is selected in query
- max[keyOrdinalList.get(i)] = Long.MAX_VALUE;
- }
- return generator.generateKey(max);
- }
-
- /**
- * To get the max key based on dimensions. i.e. all other dimensions will be
- * set to 0 bits and the required query dimension will be masked with all
- * LONG.MAX so that we can mask key and then compare while aggregating This
- * can be useful during filter query when only few dimensions were selected
- * out of row group
- *
- * @param queryDimensions dimension selected in query
- * @param generator key generator
- * @return max key for dimension
- * @throws KeyGenException if any problem while generating the key
- */
- public static byte[] getMaxKeyBasedOnDimensions(List<QueryDimension> queryDimensions,
- KeyGenerator generator) throws KeyGenException {
- long[] max = new long[generator.getDimCount()];
- Arrays.fill(max, 0L);
-
- for (int i = 0; i < queryDimensions.size(); i++) {
- // as no dictionary column and complex type columns
- // are not selected in the mdkey
- // so we will not select the those dimension for calculating the
- // range
- if (queryDimensions.get(i).getDimension().getKeyOrdinal() == -1) {
- continue;
- }
- // adding for dimension which is selected in query
- max[queryDimensions.get(i).getDimension().getKeyOrdinal()] = Long.MAX_VALUE;
- }
-
- return generator.generateKey(max);
- }
-
- /**
- * Below method will be used to get the masked key for query
- *
- * @param keySize size of the masked key
- * @param maskedKeyRanges masked byte range
- * @return masked bytes
- */
- public static int[] getMaskedByte(int keySize, int[] maskedKeyRanges) {
- int[] maskedKey = new int[keySize];
- // all the non selected dimension will be filled with -1
- Arrays.fill(maskedKey, -1);
- for (int i = 0; i < maskedKeyRanges.length; i++) {
- maskedKey[maskedKeyRanges[i]] = i;
- }
- return maskedKey;
- }
-
- /**
- * Below method will be used to get the dimension block index in file based
- * on query dimension
- *
- * @param queryDimensions query dimension
- * @param dimensionOrdinalToBlockMapping mapping of dimension block in file to query dimension
- * @return block index of file
- */
- public static int[] getDimensionsBlockIndexes(List<QueryDimension> queryDimensions,
- Map<Integer, Integer> dimensionOrdinalToBlockMapping,
- List<CarbonDimension> customAggregationDimension, Set<CarbonDimension> filterDimensions,
- List<Integer> allProjectionListDimensionIndexes) {
- // using set as in row group columns will point to same block
- Set<Integer> dimensionBlockIndex = new HashSet<Integer>();
- Set<Integer> filterDimensionOrdinal = getFilterDimensionOrdinal(filterDimensions);
- int blockIndex = 0;
- for (int i = 0; i < queryDimensions.size(); i++) {
- if (queryDimensions.get(i).getDimension().hasEncoding(Encoding.IMPLICIT)) {
- continue;
- }
- allProjectionListDimensionIndexes.add(queryDimensions.get(i).getDimension().getOrdinal());
-
- if (!filterDimensionOrdinal.contains(queryDimensions.get(i).getDimension().getOrdinal())) {
- blockIndex =
- dimensionOrdinalToBlockMapping.get(queryDimensions.get(i).getDimension().getOrdinal());
- dimensionBlockIndex.add(blockIndex);
- if (queryDimensions.get(i).getDimension().numberOfChild() > 0) {
- addChildrenBlockIndex(dimensionBlockIndex, queryDimensions.get(i).getDimension());
- }
- }
- }
- for (int i = 0; i < customAggregationDimension.size(); i++) {
- blockIndex =
- dimensionOrdinalToBlockMapping.get(customAggregationDimension.get(i).getOrdinal());
- // not adding the children dimension as dimension aggregation
- // is not push down in case of complex dimension
- dimensionBlockIndex.add(blockIndex);
- }
- int[] dimensionIndex = ArrayUtils
- .toPrimitive(dimensionBlockIndex.toArray(new Integer[dimensionBlockIndex.size()]));
- Arrays.sort(dimensionIndex);
- return dimensionIndex;
- }
-
- /**
- * Below method will be used to add the children block index
- * this will be basically for complex dimension which will have children
- *
- * @param blockIndexes block indexes
- * @param dimension parent dimension
- */
- private static void addChildrenBlockIndex(Set<Integer> blockIndexes, CarbonDimension dimension) {
- for (int i = 0; i < dimension.numberOfChild(); i++) {
- addChildrenBlockIndex(blockIndexes, dimension.getListOfChildDimensions().get(i));
- blockIndexes.add(dimension.getListOfChildDimensions().get(i).getOrdinal());
- }
- }
-
- /**
- * Below method will be used to get the dictionary mapping for all the
- * dictionary encoded dimension present in the query
- *
- * @param queryDimensions query dimension present in the query this will be used to
- * convert the result from surrogate key to actual data
- * @param absoluteTableIdentifier absolute table identifier
- * @return dimension unique id to its dictionary map
- * @throws IOException
- */
- public static Map<String, Dictionary> getDimensionDictionaryDetail(
- List<QueryDimension> queryDimensions, Set<CarbonDimension> filterComplexDimensions,
- AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException {
- // to store dimension unique column id list, this is required as
- // dimension can be present in
- // query dimension, as well as some aggregation function will be applied
- // in the same dimension
- // so we need to get only one instance of dictionary
- // direct dictionary skip is done only for the dictionary lookup
- Set<String> dictionaryDimensionFromQuery = new HashSet<String>();
- for (int i = 0; i < queryDimensions.size(); i++) {
- List<Encoding> encodingList = queryDimensions.get(i).getDimension().getEncoder();
- // TODO need to remove the data type check for parent column in complex type no need to
- // write encoding dictionary
- if (CarbonUtil.hasEncoding(encodingList, Encoding.DICTIONARY) && !CarbonUtil
- .hasEncoding(encodingList, Encoding.DIRECT_DICTIONARY) && !CarbonUtil
- .hasEncoding(encodingList, Encoding.IMPLICIT)) {
-
- if (queryDimensions.get(i).getDimension().numberOfChild() == 0) {
- dictionaryDimensionFromQuery.add(queryDimensions.get(i).getDimension().getColumnId());
- }
- if (queryDimensions.get(i).getDimension().numberOfChild() > 0) {
- getChildDimensionDictionaryDetail(queryDimensions.get(i).getDimension(),
- dictionaryDimensionFromQuery);
- }
- }
- }
- Iterator<CarbonDimension> iterator = filterComplexDimensions.iterator();
- while (iterator.hasNext()) {
- getChildDimensionDictionaryDetail(iterator.next(), dictionaryDimensionFromQuery);
- }
- // converting to list as api exposed needed list which i think
- // is not correct
- List<String> dictionaryColumnIdList =
- new ArrayList<String>(dictionaryDimensionFromQuery.size());
- dictionaryColumnIdList.addAll(dictionaryDimensionFromQuery);
- return getDictionaryMap(dictionaryColumnIdList, absoluteTableIdentifier);
- }
-
- /**
- * Below method will be used to fill the children dimension column id
- *
- * @param queryDimensions query dimension
- * @param dictionaryDimensionFromQuery dictionary dimension for query
- */
- private static void getChildDimensionDictionaryDetail(CarbonDimension queryDimensions,
- Set<String> dictionaryDimensionFromQuery) {
- for (int j = 0; j < queryDimensions.numberOfChild(); j++) {
- List<Encoding> encodingList = queryDimensions.getListOfChildDimensions().get(j).getEncoder();
- if (queryDimensions.getListOfChildDimensions().get(j).numberOfChild() > 0) {
- getChildDimensionDictionaryDetail(queryDimensions.getListOfChildDimensions().get(j),
- dictionaryDimensionFromQuery);
- } else if (!CarbonUtil.hasEncoding(encodingList, Encoding.DIRECT_DICTIONARY)) {
- dictionaryDimensionFromQuery
- .add(queryDimensions.getListOfChildDimensions().get(j).getColumnId());
- }
- }
- }
-
- /**
- * Below method will be used to get the column id to its dictionary mapping
- *
- * @param dictionaryColumnIdList dictionary column list
- * @param absoluteTableIdentifier absolute table identifier
- * @return dictionary mapping
- * @throws IOException
- */
- private static Map<String, Dictionary> getDictionaryMap(List<String> dictionaryColumnIdList,
- AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException {
- // this for dictionary unique identifier
- List<DictionaryColumnUniqueIdentifier> dictionaryColumnUniqueIdentifiers =
- getDictionaryColumnUniqueIdentifierList(dictionaryColumnIdList,
- absoluteTableIdentifier.getCarbonTableIdentifier());
- CacheProvider cacheProvider = CacheProvider.getInstance();
- Cache<DictionaryColumnUniqueIdentifier, Dictionary> forwardDictionaryCache = cacheProvider
- .createCache(CacheType.FORWARD_DICTIONARY, absoluteTableIdentifier.getStorePath());
-
- List<Dictionary> columnDictionaryList =
- forwardDictionaryCache.getAll(dictionaryColumnUniqueIdentifiers);
- Map<String, Dictionary> columnDictionaryMap = new HashMap<>(columnDictionaryList.size());
- for (int i = 0; i < dictionaryColumnUniqueIdentifiers.size(); i++) {
- // TODO: null check for column dictionary, if cache size is less it
- // might return null here, in that case throw exception
- columnDictionaryMap.put(dictionaryColumnIdList.get(i), columnDictionaryList.get(i));
- }
- return columnDictionaryMap;
- }
-
- /**
- * Below method will be used to get the dictionary column unique identifier
- *
- * @param dictionaryColumnIdList dictionary
- * @param carbonTableIdentifier
- * @return
- */
- private static List<DictionaryColumnUniqueIdentifier> getDictionaryColumnUniqueIdentifierList(
- List<String> dictionaryColumnIdList, CarbonTableIdentifier carbonTableIdentifier) {
- CarbonTable carbonTable =
- CarbonMetadata.getInstance().getCarbonTable(carbonTableIdentifier.getTableUniqueName());
- List<DictionaryColumnUniqueIdentifier> dictionaryColumnUniqueIdentifiers =
- new ArrayList<>(dictionaryColumnIdList.size());
- for (String columnId : dictionaryColumnIdList) {
- CarbonDimension dimension = CarbonMetadata.getInstance()
- .getCarbonDimensionBasedOnColIdentifier(carbonTable, columnId);
- if (dimension != null) {
- dictionaryColumnUniqueIdentifiers.add(
- new DictionaryColumnUniqueIdentifier(
- carbonTableIdentifier,
- dimension.getColumnIdentifier(),
- dimension.getDataType()
- )
- );
- }
- }
- return dictionaryColumnUniqueIdentifiers;
- }
-
- /**
- * Below method will used to get the method will be used to get the measure
- * block indexes to be read from the file
- *
- * @param queryMeasures query measure
- * @param expressionMeasure measure present in the expression
- * @param ordinalToBlockIndexMapping measure ordinal to block mapping
- * @return block indexes
- */
- public static int[] getMeasureBlockIndexes(List<QueryMeasure> queryMeasures,
- List<CarbonMeasure> expressionMeasure, Map<Integer, Integer> ordinalToBlockIndexMapping,
- Set<CarbonMeasure> filterMeasures) {
- Set<Integer> measureBlockIndex = new HashSet<Integer>();
- Set<Integer> filterMeasureOrdinal = getFilterMeasureOrdinal(filterMeasures);
- for (int i = 0; i < queryMeasures.size(); i++) {
- if (!filterMeasureOrdinal.contains(queryMeasures.get(i).getMeasure().getOrdinal())) {
- measureBlockIndex
- .add(ordinalToBlockIndexMapping.get(queryMeasures.get(i).getMeasure().getOrdinal()));
- }
- }
- for (int i = 0; i < expressionMeasure.size(); i++) {
- measureBlockIndex.add(ordinalToBlockIndexMapping.get(expressionMeasure.get(i).getOrdinal()));
- }
- int[] measureIndexes =
- ArrayUtils.toPrimitive(measureBlockIndex.toArray(new Integer[measureBlockIndex.size()]));
- Arrays.sort(measureIndexes);
- return measureIndexes;
- }
-
- /**
- * Below method will be used to get mapping whether dimension is present in
- * order by or not
- *
- * @param sortedDimensions sort dimension present in order by query
- * @param queryDimensions query dimension
- * @return sort dimension indexes
- */
- public static byte[] getSortDimensionIndexes(List<QueryDimension> sortedDimensions,
- List<QueryDimension> queryDimensions) {
- byte[] sortedDims = new byte[queryDimensions.size()];
- int indexOf = 0;
- for (int i = 0; i < sortedDims.length; i++) {
- indexOf = sortedDimensions.indexOf(queryDimensions.get(i));
- if (indexOf > -1) {
- sortedDims[i] = 1;
- }
- }
- return sortedDims;
- }
-
- /**
- * Below method will be used to get the mapping of block index and its
- * restructuring info
- *
- * @param queryDimensions query dimension from query model
- * @param segmentProperties segment properties
- * @return map of block index to its restructuring info
- * @throws KeyGenException if problem while key generation
- */
- public static Map<Integer, KeyStructureInfo> getColumnGroupKeyStructureInfo(
- List<QueryDimension> queryDimensions, SegmentProperties segmentProperties)
- throws KeyGenException {
- Map<Integer, KeyStructureInfo> rowGroupToItsRSInfo = new HashMap<Integer, KeyStructureInfo>();
- // get column group id and its ordinal mapping of column group
- Map<Integer, List<Integer>> columnGroupAndItsOrdinalMappingForQuery =
- getColumnGroupAndItsOrdinalMapping(queryDimensions);
- Map<Integer, KeyGenerator> columnGroupAndItsKeygenartor =
- segmentProperties.getColumnGroupAndItsKeygenartor();
-
- Iterator<Entry<Integer, List<Integer>>> iterator =
- columnGroupAndItsOrdinalMappingForQuery.entrySet().iterator();
- KeyStructureInfo restructureInfos = null;
- while (iterator.hasNext()) {
- Entry<Integer, List<Integer>> next = iterator.next();
- KeyGenerator keyGenerator = columnGroupAndItsKeygenartor.get(next.getKey());
- restructureInfos = new KeyStructureInfo();
- // sort the ordinal
- List<Integer> ordinal = next.getValue();
- List<Integer> mdKeyOrdinal = new ArrayList<Integer>();
- //Un sorted
- List<Integer> mdKeyOrdinalForQuery = new ArrayList<Integer>();
- for (Integer ord : ordinal) {
- mdKeyOrdinal.add(segmentProperties.getColumnGroupMdKeyOrdinal(next.getKey(), ord));
- mdKeyOrdinalForQuery.add(segmentProperties.getColumnGroupMdKeyOrdinal(next.getKey(), ord));
- }
- Collections.sort(mdKeyOrdinal);
- // get the masked byte range for column group
- int[] maskByteRanges = getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
- // max key for column group
- byte[] maxKey = getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator);
- restructureInfos.setKeyGenerator(keyGenerator);
- restructureInfos.setMaskByteRanges(maskByteRanges);
- restructureInfos.setMaxKey(maxKey);
- restructureInfos.setMdkeyQueryDimensionOrdinal(ArrayUtils
- .toPrimitive(mdKeyOrdinalForQuery.toArray(new Integer[mdKeyOrdinalForQuery.size()])));
- rowGroupToItsRSInfo
- .put(segmentProperties.getDimensionOrdinalToBlockMapping().get(ordinal.get(0)),
- restructureInfos);
- }
- return rowGroupToItsRSInfo;
- }
-
- /**
- * return true if given key is found in array
- *
- * @param data
- * @param key
- * @return
- */
- public static boolean searchInArray(int[] data, int key) {
- for (int i = 0; i < data.length; i++) {
- if (key == data[i]) {
- return true;
- }
- }
- return false;
- }
-
- /**
- * Below method will be used to create a mapping of column group columns
- * this mapping will have column group id to all the dimension ordinal
- * present in the column group This mapping will be used during query
- * execution, to create a mask key for the column group dimension which will
- * be used in aggregation and filter query as column group dimension will be
- * stored in bit level
- */
- private static Map<Integer, List<Integer>> getColumnGroupAndItsOrdinalMapping(
- List<QueryDimension> origdimensions) {
-
- List<QueryDimension> dimensions = new ArrayList<QueryDimension>(origdimensions.size());
- dimensions.addAll(origdimensions);
- /**
- * sort based on column group id
- */
- Collections.sort(dimensions, new Comparator<QueryDimension>() {
-
- @Override public int compare(QueryDimension o1, QueryDimension o2) {
- return Integer
- .compare(o1.getDimension().columnGroupId(), o2.getDimension().columnGroupId());
- }
- });
- // list of row groups this will store all the row group column
- Map<Integer, List<Integer>> columnGroupAndItsOrdinalsMapping =
- new HashMap<Integer, List<Integer>>();
- // to store a column group
- List<Integer> currentColumnGroup = null;
- // current index
- int index = 0;
- // previous column group to check all the column of row id has bee
- // selected
- int prvColumnGroupId = -1;
- while (index < dimensions.size()) {
- // if dimension group id is not zero and it is same as the previous
- // column group id
- // then we need to add ordinal of that column as it belongs to same
- // column group
- if (dimensions.get(index).getDimension().hasEncoding(Encoding.IMPLICIT)) {
- index++;
- continue;
- } else if (!dimensions.get(index).getDimension().isColumnar()
- && dimensions.get(index).getDimension().columnGroupId() == prvColumnGroupId
- && null != currentColumnGroup) {
- currentColumnGroup.add(dimensions.get(index).getDimension().getOrdinal());
- }
-
- // if dimension is not a columnar then it is column group column
- else if (!dimensions.get(index).getDimension().isColumnar()) {
- currentColumnGroup = new ArrayList<Integer>();
- columnGroupAndItsOrdinalsMapping
- .put(dimensions.get(index).getDimension().columnGroupId(), currentColumnGroup);
- currentColumnGroup.add(dimensions.get(index).getDimension().getOrdinal());
- }
- // update the row id every time,this is required to group the
- // columns
- // of the same row group
- prvColumnGroupId = dimensions.get(index).getDimension().columnGroupId();
- index++;
- }
- return columnGroupAndItsOrdinalsMapping;
- }
-
- /**
- * Below method will be used to get masked byte
- *
- * @param data actual data
- * @param maxKey max key
- * @param maskByteRanges mask byte range
- * @param byteCount
- * @return masked byte
- */
- public static byte[] getMaskedKey(byte[] data, byte[] maxKey, int[] maskByteRanges,
- int byteCount) {
- byte[] maskedKey = new byte[byteCount];
- int counter = 0;
- int byteRange = 0;
- for (int i = 0; i < byteCount; i++) {
- byteRange = maskByteRanges[i];
- if (byteRange != -1) {
- maskedKey[counter++] = (byte) (data[byteRange] & maxKey[byteRange]);
- }
- }
- return maskedKey;
- }
-
- /**
- * Below method will be used to fill block indexes of the query dimension
- * which will be used in creating a output row Here is method we are passing
- * two list which store the indexes one for dictionary column other for not
- * dictionary column. This is done for specific purpose so that in one
- * iteration we will be able to fill both type dimension block indexes
- *
- * @param queryDimensions dimension present in the query
- * @param columnOrdinalToBlockIndexMapping column ordinal to block index mapping
- * @param dictionaryDimensionBlockIndex list to store dictionary column block indexes
- * @param noDictionaryDimensionBlockIndex list to store no dictionary block indexes
- */
- public static void fillQueryDimensionsBlockIndexes(List<QueryDimension> queryDimensions,
- Map<Integer, Integer> columnOrdinalToBlockIndexMapping,
- Set<Integer> dictionaryDimensionBlockIndex, List<Integer> noDictionaryDimensionBlockIndex) {
- for (QueryDimension queryDimension : queryDimensions) {
- if (CarbonUtil.hasEncoding(queryDimension.getDimension().getEncoder(), Encoding.DICTIONARY)
- && queryDimension.getDimension().numberOfChild() == 0) {
- dictionaryDimensionBlockIndex
- .add(columnOrdinalToBlockIndexMapping.get(queryDimension.getDimension().getOrdinal()));
- } else if (
- !CarbonUtil.hasEncoding(queryDimension.getDimension().getEncoder(), Encoding.IMPLICIT)
- && queryDimension.getDimension().numberOfChild() == 0) {
- noDictionaryDimensionBlockIndex
- .add(columnOrdinalToBlockIndexMapping.get(queryDimension.getDimension().getOrdinal()));
- }
- }
- }
-
- /**
- * Below method will be used to resolve the query model
- * resolve will be setting the actual dimension and measure object
- * as from driver only column name will be passes to avoid the heavy object
- * serialization
- *
- * @param queryModel query model
- */
- public static void resolveQueryModel(QueryModel queryModel) {
- CarbonMetadata.getInstance().addCarbonTable(queryModel.getTable());
- // TODO need to load the table from table identifier
- CarbonTable carbonTable = queryModel.getTable();
- String tableName =
- queryModel.getAbsoluteTableIdentifier().getCarbonTableIdentifier().getTableName();
- // resolve query dimension
- for (QueryDimension queryDimension : queryModel.getQueryDimension()) {
- queryDimension
- .setDimension(carbonTable.getDimensionByName(tableName, queryDimension.getColumnName()));
- }
- // resolve query measure
- for (QueryMeasure queryMeasure : queryModel.getQueryMeasures()) {
- // in case of count start column name will be count * so
- // first need to check any measure is present or not and as if measure
- // if measure is present and if first measure is not a default
- // measure than add measure otherwise
- // than add first dimension as a measure
- //as currently if measure is not present then
- //we are adding default measure so first condition will
- //never come false but if in future we can remove so not removing first if check
- if (queryMeasure.getColumnName().equals("count(*)")) {
- if (carbonTable.getMeasureByTableName(tableName).size() > 0 && !carbonTable
- .getMeasureByTableName(tableName).get(0).getColName()
- .equals(CarbonCommonConstants.DEFAULT_INVISIBLE_DUMMY_MEASURE)) {
- queryMeasure.setMeasure(carbonTable.getMeasureByTableName(tableName).get(0));
- } else {
- CarbonMeasure dummyMeasure = new CarbonMeasure(
- carbonTable.getDimensionByTableName(tableName).get(0).getColumnSchema(), 0);
- queryMeasure.setMeasure(dummyMeasure);
- }
- } else {
- queryMeasure
- .setMeasure(carbonTable.getMeasureByName(tableName, queryMeasure.getColumnName()));
- }
- }
- }
-
- /**
- * below method will be used to get the actual type aggregator
- *
- * @param aggType
- * @return index in aggrgetor
- */
- public static int[] getActualTypeIndex(List<String> aggType) {
- List<Integer> indexList = new ArrayList<Integer>();
- for (int i = 0; i < aggType.size(); i++) {
- if (!CarbonCommonConstants.SUM.equals(aggType.get(i)) && !CarbonCommonConstants.AVERAGE
- .equals(aggType.get(i))) {
- indexList.add(i);
- }
- }
- return ArrayUtils.toPrimitive(indexList.toArray(new Integer[indexList.size()]));
- }
-
- /**
- * Below method will be used to get the key structure for the column group
- *
- * @param segmentProperties segment properties
- * @param dimColumnEvaluatorInfo dimension evaluator info
- * @return key structure info for column group dimension
- * @throws KeyGenException
- */
- public static KeyStructureInfo getKeyStructureInfo(SegmentProperties segmentProperties,
- DimColumnResolvedFilterInfo dimColumnEvaluatorInfo) throws KeyGenException {
- int colGrpId = getColumnGroupId(segmentProperties, dimColumnEvaluatorInfo.getColumnIndex());
- KeyGenerator keyGenerator = segmentProperties.getColumnGroupAndItsKeygenartor().get(colGrpId);
- List<Integer> mdKeyOrdinal = new ArrayList<Integer>();
-
- mdKeyOrdinal.add(segmentProperties
- .getColumnGroupMdKeyOrdinal(colGrpId, dimColumnEvaluatorInfo.getColumnIndex()));
- int[] maskByteRanges = QueryUtil.getMaskedByteRangeBasedOrdinal(mdKeyOrdinal, keyGenerator);
- byte[] maxKey = QueryUtil.getMaxKeyBasedOnOrinal(mdKeyOrdinal, keyGenerator);
- KeyStructureInfo restructureInfos = new KeyStructureInfo();
- restructureInfos.setKeyGenerator(keyGenerator);
- restructureInfos.setMaskByteRanges(maskByteRanges);
- restructureInfos.setMaxKey(maxKey);
- return restructureInfos;
- }
-
- /**
- * Below method will be used to get the column group id based on the ordinal
- *
- * @param segmentProperties segment properties
- * @param ordinal ordinal to be searched
- * @return column group id
- */
- public static int getColumnGroupId(SegmentProperties segmentProperties, int ordinal) {
- int[][] columnGroups = segmentProperties.getColumnGroups();
- int colGrpId = -1;
- for (int i = 0; i < columnGroups.length; i++) {
- if (columnGroups[i].length > 1) {
- colGrpId++;
- if (QueryUtil.searchInArray(columnGroups[i], ordinal)) {
- break;
- }
- }
- }
- return colGrpId;
- }
-
- /**
- * Below method will be used to get the map of for complex dimension and its type
- * which will be used to during query execution to
- *
- * @param queryDimensions complex dimension in query
- * @param dimensionToBlockIndexMap dimension to block index in file map
- * @return complex dimension and query type
- */
- public static Map<Integer, GenericQueryType> getComplexDimensionsMap(
- List<QueryDimension> queryDimensions, Map<Integer, Integer> dimensionToBlockIndexMap,
- int[] eachComplexColumnValueSize, Map<String, Dictionary> columnIdToDictionaryMap,
- Set<CarbonDimension> filterDimensions) {
- Map<Integer, GenericQueryType> complexTypeMap = new HashMap<Integer, GenericQueryType>();
- for (QueryDimension dimension : queryDimensions) {
- CarbonDimension actualDimension = dimension.getDimension();
- if (actualDimension.getNumberOfChild() == 0) {
- continue;
- }
- fillParentDetails(dimensionToBlockIndexMap, actualDimension, complexTypeMap,
- eachComplexColumnValueSize, columnIdToDictionaryMap);
- }
- if (null != filterDimensions) {
- for (CarbonDimension filterDimension : filterDimensions) {
- // do not fill nay details for implicit dimension type
- if (filterDimension.hasEncoding(Encoding.IMPLICIT)) {
- continue;
- }
- fillParentDetails(dimensionToBlockIndexMap, filterDimension, complexTypeMap,
- eachComplexColumnValueSize, columnIdToDictionaryMap);
- }
- }
- return complexTypeMap;
- }
-
- private static void fillParentDetails(Map<Integer, Integer> dimensionToBlockIndexMap,
- CarbonDimension dimension, Map<Integer, GenericQueryType> complexTypeMap,
- int[] eachComplexColumnValueSize, Map<String, Dictionary> columnIdToDictionaryMap) {
- int parentBlockIndex = dimensionToBlockIndexMap.get(dimension.getOrdinal());
- GenericQueryType parentQueryType = dimension.getDataType().equals(DataType.ARRAY) ?
- new ArrayQueryType(dimension.getColName(), dimension.getColName(), parentBlockIndex) :
- new StructQueryType(dimension.getColName(), dimension.getColName(),
- dimensionToBlockIndexMap.get(dimension.getOrdinal()));
- complexTypeMap.put(dimension.getOrdinal(), parentQueryType);
- parentBlockIndex =
- fillChildrenDetails(eachComplexColumnValueSize, columnIdToDictionaryMap, parentBlockIndex,
- dimension, parentQueryType);
- }
-
- private static int fillChildrenDetails(int[] eachComplexColumnValueSize,
- Map<String, Dictionary> columnIdToDictionaryMap, int parentBlockIndex,
- CarbonDimension dimension, GenericQueryType parentQueryType) {
- for (int i = 0; i < dimension.getNumberOfChild(); i++) {
- switch (dimension.getListOfChildDimensions().get(i).getDataType()) {
- case ARRAY:
- parentQueryType.addChildren(
- new ArrayQueryType(dimension.getListOfChildDimensions().get(i).getColName(),
- dimension.getColName(), ++parentBlockIndex));
- break;
- case STRUCT:
- parentQueryType.addChildren(
- new StructQueryType(dimension.getListOfChildDimensions().get(i).getColName(),
- dimension.getColName(), ++parentBlockIndex));
- break;
- default:
- boolean isDirectDictionary = CarbonUtil
- .hasEncoding(dimension.getListOfChildDimensions().get(i).getEncoder(),
- Encoding.DIRECT_DICTIONARY);
- parentQueryType.addChildren(
- new PrimitiveQueryType(dimension.getListOfChildDimensions().get(i).getColName(),
- dimension.getColName(), ++parentBlockIndex,
- dimension.getListOfChildDimensions().get(i).getDataType(),
- eachComplexColumnValueSize[dimension.getListOfChildDimensions().get(i)
- .getComplexTypeOrdinal()], columnIdToDictionaryMap
- .get(dimension.getListOfChildDimensions().get(i).getColumnId()),
- isDirectDictionary));
- }
- if (dimension.getListOfChildDimensions().get(i).getNumberOfChild() > 0) {
- parentBlockIndex = fillChildrenDetails(eachComplexColumnValueSize, columnIdToDictionaryMap,
- parentBlockIndex, dimension.getListOfChildDimensions().get(i), parentQueryType);
- }
- }
- return parentBlockIndex;
- }
-
- public static void getAllFilterDimensions(FilterResolverIntf filterResolverTree,
- Set<CarbonDimension> filterDimensions, Set<CarbonMeasure> filterMeasure) {
- if (null == filterResolverTree) {
- return;
- }
- List<ColumnExpression> dimensionResolvedInfos = new ArrayList<ColumnExpression>();
- Expression filterExpression = filterResolverTree.getFilterExpression();
- addColumnDimensions(filterExpression, filterDimensions, filterMeasure);
- for (ColumnExpression info : dimensionResolvedInfos) {
- if (info.isDimension() && info.getDimension().getNumberOfChild() > 0) {
- filterDimensions.add(info.getDimension());
- }
- }
- }
-
- /**
- * This method will check if a given expression contains a column expression
- * recursively and add the dimension instance to the set which holds the dimension
- * instances of the complex filter expressions.
- */
- private static void addColumnDimensions(Expression expression,
- Set<CarbonDimension> filterDimensions, Set<CarbonMeasure> filterMeasure) {
- if (null != expression && expression instanceof ColumnExpression) {
- if (((ColumnExpression) expression).isDimension()) {
- filterDimensions.add(((ColumnExpression) expression).getDimension());
- } else {
- filterMeasure.add((CarbonMeasure) ((ColumnExpression) expression).getCarbonColumn());
- }
- return;
- } else if (null != expression) {
- for (Expression child : expression.getChildren()) {
- addColumnDimensions(child, filterDimensions, filterMeasure);
- }
- }
- }
-
- private static Set<Integer> getFilterMeasureOrdinal(Set<CarbonMeasure> filterMeasures) {
- Set<Integer> filterMeasuresOrdinal = new HashSet<>();
- for (CarbonMeasure filterMeasure : filterMeasures) {
- filterMeasuresOrdinal.add(filterMeasure.getOrdinal());
- }
- return filterMeasuresOrdinal;
- }
-
- private static Set<Integer> getFilterDimensionOrdinal(Set<CarbonDimension> filterDimensions) {
- Set<Integer> filterDimensionsOrdinal = new HashSet<>();
- for (CarbonDimension filterDimension : filterDimensions) {
- filterDimensionsOrdinal.add(filterDimension.getOrdinal());
- getChildDimensionOrdinal(filterDimension, filterDimensionsOrdinal);
- }
- return filterDimensionsOrdinal;
- }
-
- /**
- * Below method will be used to fill the children dimension column id
- */
- private static void getChildDimensionOrdinal(CarbonDimension queryDimensions,
- Set<Integer> filterDimensionsOrdinal) {
- for (int j = 0; j < queryDimensions.numberOfChild(); j++) {
- List<Encoding> encodingList = queryDimensions.getListOfChildDimensions().get(j).getEncoder();
- if (queryDimensions.getListOfChildDimensions().get(j).numberOfChild() > 0) {
- getChildDimensionOrdinal(queryDimensions.getListOfChildDimensions().get(j),
- filterDimensionsOrdinal);
- } else if (!CarbonUtil.hasEncoding(encodingList, Encoding.DIRECT_DICTIONARY)) {
- filterDimensionsOrdinal.add(queryDimensions.getListOfChildDimensions().get(j).getOrdinal());
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/executor/util/RestructureUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/util/RestructureUtil.java b/core/src/main/java/org/apache/carbondata/scan/executor/util/RestructureUtil.java
deleted file mode 100644
index c5daa17..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/executor/util/RestructureUtil.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.executor.util;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.core.metadata.Encoding;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.scan.executor.infos.AggregatorInfo;
-import org.apache.carbondata.scan.model.QueryDimension;
-import org.apache.carbondata.scan.model.QueryMeasure;
-
-/**
- * Utility class for restructuring
- */
-public class RestructureUtil {
-
- /**
- * Below method will be used to get the updated query dimension updation
- * means, after restructuring some dimension will be not present in older
- * table blocks in that case we need to select only those dimension out of
- * query dimension which is present in the current table block
- *
- * @param queryDimensions
- * @param tableBlockDimensions
- * @return list of query dimension which is present in the table block
- */
- public static List<QueryDimension> getUpdatedQueryDimension(List<QueryDimension> queryDimensions,
- List<CarbonDimension> tableBlockDimensions, List<CarbonDimension> tableComplexDimension) {
- List<QueryDimension> presentDimension =
- new ArrayList<QueryDimension>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- // selecting only those dimension which is present in the query
- for (QueryDimension queryDimension : queryDimensions) {
- if (queryDimension.getDimension().hasEncoding(Encoding.IMPLICIT)) {
- presentDimension.add(queryDimension);
- } else {
- for (CarbonDimension tableDimension : tableBlockDimensions) {
- if (tableDimension.equals(queryDimension.getDimension())) {
- presentDimension.add(queryDimension);
- }
- }
- }
- }
- for (QueryDimension queryDimimension : queryDimensions) {
- for (CarbonDimension tableDimension : tableComplexDimension) {
- if (tableDimension.equals(queryDimimension.getDimension())) {
- presentDimension.add(queryDimimension);
- }
- }
- }
- return presentDimension;
- }
-
- /**
- * Below method is to add dimension children for complex type dimension as
- * internally we are creating dimension column for each each complex
- * dimension so when complex query dimension request will come in the query,
- * we need to add its children as it is hidden from the user For example if
- * complex dimension is of Array of String[2] so we are storing 3 dimension
- * and when user will query for complex type i.e. array type we need to add
- * its children and then we will read respective block and create a tuple
- * based on all three dimension
- *
- * @param queryDimensions current query dimensions
- * @param tableBlockDimensions dimensions which is present in the table block
- * @return updated dimension(after adding complex type children)
- */
- public static List<CarbonDimension> addChildrenForComplexTypeDimension(
- List<CarbonDimension> queryDimensions, List<CarbonDimension> tableBlockDimensions) {
- List<CarbonDimension> updatedQueryDimension = new ArrayList<CarbonDimension>();
- int numberOfChildren = 0;
- for (CarbonDimension queryDimension : queryDimensions) {
- // if number of child is zero, then it is not a complex dimension
- // so directly add it query dimension
- if (queryDimension.numberOfChild() == 0) {
- updatedQueryDimension.add(queryDimension);
- }
- // if number of child is more than 1 then add all its children
- numberOfChildren = queryDimension.getOrdinal() + queryDimension.numberOfChild();
- for (int j = queryDimension.getOrdinal(); j < numberOfChildren; j++) {
- updatedQueryDimension.add(tableBlockDimensions.get(j));
- }
- }
- return updatedQueryDimension;
- }
-
- /**
- * Below method will be used to get the aggregator info object
- * in this method some of the properties which will be extracted
- * from query measure and current block measures will be set
- *
- * @param queryMeasures measures present in query
- * @param currentBlockMeasures current block measures
- * @return aggregator info
- */
- public static AggregatorInfo getAggregatorInfos(List<QueryMeasure> queryMeasures,
- List<CarbonMeasure> currentBlockMeasures) {
- AggregatorInfo aggregatorInfos = new AggregatorInfo();
- int numberOfMeasureInQuery = queryMeasures.size();
- int[] measureOrdinals = new int[numberOfMeasureInQuery];
- Object[] defaultValues = new Object[numberOfMeasureInQuery];
- boolean[] measureExistsInCurrentBlock = new boolean[numberOfMeasureInQuery];
- int index = 0;
- for (QueryMeasure queryMeasure : queryMeasures) {
- measureOrdinals[index] = queryMeasure.getMeasure().getOrdinal();
- // if query measure exists in current dimension measures
- // then setting measure exists is true
- // otherwise adding a default value of a measure
- if (currentBlockMeasures.contains(queryMeasure.getMeasure())) {
- measureExistsInCurrentBlock[index] = true;
- } else {
- defaultValues[index] = queryMeasure.getMeasure().getDefaultValue();
- }
- index++;
- }
- aggregatorInfos.setDefaultValues(defaultValues);
- aggregatorInfos.setMeasureOrdinals(measureOrdinals);
- aggregatorInfos.setMeasureExists(measureExistsInCurrentBlock);
- return aggregatorInfos;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/expression/BinaryExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/BinaryExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/BinaryExpression.java
deleted file mode 100644
index e50525d..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/expression/BinaryExpression.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression;
-
-public abstract class BinaryExpression extends Expression {
-
- private static final long serialVersionUID = 1L;
- protected Expression left;
- protected Expression right;
-
- public BinaryExpression(Expression left, Expression right) {
- this.left = left;
- this.right = right;
- children.add(left);
- children.add(right);
- }
-
- public Expression getLeft() {
- return left;
- }
-
- public Expression getRight() {
- return right;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/expression/ColumnExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/ColumnExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/ColumnExpression.java
deleted file mode 100644
index 4e19308..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/expression/ColumnExpression.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.scan.filter.intf.ExpressionType;
-import org.apache.carbondata.scan.filter.intf.RowIntf;
-
-public class ColumnExpression extends LeafExpression {
-
- private static final long serialVersionUID = 1L;
-
- private String columnName;
-
- private boolean isDimension;
-
- private int colIndex = -1;
-
- private DataType dataType;
-
- private CarbonDimension dimension;
-
- private CarbonColumn carbonColumn;
-
- public ColumnExpression(String columnName, DataType dataType) {
- this.columnName = columnName;
- this.dataType = dataType;
-
- }
-
- public CarbonDimension getDimension() {
- return dimension;
- }
-
- public void setDimension(CarbonDimension dimension) {
- this.dimension = dimension;
- }
-
- public String getColumnName() {
- return columnName;
- }
-
- public void setColumnName(String columnName) {
- this.columnName = columnName;
- }
-
- public boolean isDimension() {
- return isDimension;
- }
-
- public void setDimension(boolean isDimension) {
- this.isDimension = isDimension;
- }
-
- public int getColIndex() {
- return colIndex;
- }
-
- public void setColIndex(int colIndex) {
- this.colIndex = colIndex;
- }
-
- public DataType getDataType() {
- return dataType;
- }
-
- public void setDataType(DataType dataType) {
- this.dataType = dataType;
- }
-
- @Override public ExpressionResult evaluate(RowIntf value) {
- ExpressionResult expressionResult =
- new ExpressionResult(dataType, (null == value ? null : value.getVal(colIndex)));
- return expressionResult;
- }
-
- @Override public ExpressionType getFilterExpressionType() {
- // TODO Auto-generated method stub
- return null;
- }
-
- @Override public String getString() {
- // TODO Auto-generated method stub
- return "ColumnExpression(" + columnName + ')';
- }
-
- public CarbonColumn getCarbonColumn() {
- return carbonColumn;
- }
-
- public void setCarbonColumn(CarbonColumn carbonColumn) {
- this.carbonColumn = carbonColumn;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/expression/Expression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/Expression.java b/core/src/main/java/org/apache/carbondata/scan/expression/Expression.java
deleted file mode 100644
index 06f6c72..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/expression/Expression.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.intf.ExpressionType;
-import org.apache.carbondata.scan.filter.intf.RowIntf;
-
-public abstract class Expression implements Serializable {
-
- private static final long serialVersionUID = -7568676723039530713L;
- protected List<Expression> children =
- new ArrayList<Expression>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-
- public abstract ExpressionResult evaluate(RowIntf value)
- throws FilterUnsupportedException, FilterIllegalMemberException;
-
- public abstract ExpressionType getFilterExpressionType();
-
- public List<Expression> getChildren() {
- return children;
- }
-
- public abstract String getString();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/expression/ExpressionResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/ExpressionResult.java b/core/src/main/java/org/apache/carbondata/scan/expression/ExpressionResult.java
deleted file mode 100644
index 5e31693..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/expression/ExpressionResult.java
+++ /dev/null
@@ -1,545 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additiona l information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression;
-
-import java.math.BigDecimal;
-import java.sql.Timestamp;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-
-public class ExpressionResult implements Comparable<ExpressionResult> {
-
- protected DataType dataType;
-
- protected Object value;
-
- private List<ExpressionResult> expressionResults;
-
- private boolean isLiteral = false;
-
- public ExpressionResult(DataType dataType, Object value) {
- this.dataType = dataType;
- this.value = value;
- }
-
- public ExpressionResult(DataType dataType, Object value, boolean isLiteral) {
- this(dataType, value);
- this.isLiteral = isLiteral;
- }
-
- public ExpressionResult(List<ExpressionResult> expressionResults) {
- this.expressionResults = expressionResults;
- }
-
- public void set(DataType dataType, Object value) {
- this.dataType = dataType;
- this.value = value;
- this.expressionResults = null;
- }
-
- public DataType getDataType() {
- return dataType;
- }
-
- public Integer getInt() throws FilterIllegalMemberException {
- if (value == null) {
- return null;
- }
- try {
- switch (this.getDataType()) {
- case STRING:
- try {
- return Integer.parseInt(value.toString());
- } catch (NumberFormatException e) {
- throw new FilterIllegalMemberException(e);
- }
- case SHORT:
- return ((Short) value).intValue();
- case INT:
- case DOUBLE:
- if (value instanceof Double) {
- return ((Double) value).intValue();
- }
- if (value instanceof Long) {
- return ((Long) value).intValue();
- }
- return (Integer) value;
- case DATE:
- if (value instanceof java.sql.Date) {
- return (int) (((java.sql.Date) value).getTime());
- } else {
- return (Integer) value;
- }
- case TIMESTAMP:
- if (value instanceof Timestamp) {
- return (int) (((Timestamp) value).getTime());
- } else {
- if (isLiteral) {
- Long l = (Long) value /1000;
- return l.intValue();
- }
- return (Integer) value;
- }
- default:
- throw new FilterIllegalMemberException(
- "Cannot convert" + this.getDataType().name() + " to integer type value");
- }
-
- } catch (ClassCastException e) {
- throw new FilterIllegalMemberException(
- "Cannot convert" + this.getDataType().name() + " to Integer type value");
- }
- }
-
- public Short getShort() throws FilterIllegalMemberException {
- if (value == null) {
- return null;
- }
- try {
- switch (this.getDataType()) {
- case STRING:
- try {
- return Short.parseShort(value.toString());
- } catch (NumberFormatException e) {
- throw new FilterIllegalMemberException(e);
- }
- case SHORT:
- case INT:
- case DOUBLE:
-
- if (value instanceof Double) {
- return ((Double) value).shortValue();
- } else if (value instanceof Integer) {
- return ((Integer) value).shortValue();
- }
- return (Short) value;
-
- case DATE:
-
- if (value instanceof java.sql.Date) {
- return (short) (((java.sql.Date) value).getTime());
- } else {
- return (Short) value;
- }
- case TIMESTAMP:
-
- if (value instanceof Timestamp) {
- return (short) (((Timestamp) value).getTime());
- } else {
- if (isLiteral) {
- Long l = ((long)value/1000);
- return l.shortValue();
- }
- return (Short) value;
- }
-
- default:
- throw new FilterIllegalMemberException(
- "Cannot convert" + this.getDataType().name() + " to integer type value");
- }
-
- } catch (ClassCastException e) {
- throw new FilterIllegalMemberException(
- "Cannot convert" + this.getDataType().name() + " to Integer type value");
- }
- }
-
- public String getString() throws FilterIllegalMemberException {
- if (value == null) {
- return null;
- }
- try {
- switch (this.getDataType()) {
- case DATE:
- case TIMESTAMP:
- SimpleDateFormat parser = new SimpleDateFormat(CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
- if (value instanceof Timestamp) {
- return parser.format((Timestamp) value);
- } else if (value instanceof java.sql.Date) {
- return parser.format((java.sql.Date) value);
- } else if (value instanceof Long) {
- if (isLiteral) {
- return parser.format(new Timestamp((long) value/1000));
- }
- return parser.format(new Timestamp((long) value));
- } else if (value instanceof Integer) {
- return parser.format(new java.sql.Date((int)value));
- }
- return value.toString();
- default:
- return value.toString();
- }
- } catch (Exception e) {
- throw new FilterIllegalMemberException(
- "Cannot convert" + this.getDataType().name() + " to String type value");
- }
- }
-
- public Double getDouble() throws FilterIllegalMemberException {
- if (value == null) {
- return null;
- }
- try {
- switch (this.getDataType()) {
- case STRING:
- try {
- return Double.parseDouble(value.toString());
- } catch (NumberFormatException e) {
- throw new FilterIllegalMemberException(e);
- }
- case SHORT:
- return ((Short) value).doubleValue();
- case INT:
- return ((Integer) value).doubleValue();
- case LONG:
- return ((Long) value).doubleValue();
- case DOUBLE:
- return (Double) value;
- case DATE:
- if (value instanceof java.sql.Date) {
- return (double) ((java.sql.Date) value).getTime();
- } else {
- return (Double) (value);
- }
- case TIMESTAMP:
- if (value instanceof Timestamp) {
- return (double) ((Timestamp) value).getTime();
- } else {
- if (isLiteral) {
- Long l = (Long) value/1000;
- return l.doubleValue();
- }
- return (Double) (value);
- }
- default:
- throw new FilterIllegalMemberException(
- "Cannot convert" + this.getDataType().name() + " to double type value");
- }
- } catch (ClassCastException e) {
- throw new FilterIllegalMemberException(
- "Cannot convert" + this.getDataType().name() + " to Double type value");
- }
- }
-
- public Long getLong() throws FilterIllegalMemberException {
- if (value == null) {
- return null;
- }
- try {
- switch (this.getDataType()) {
- case STRING:
- try {
- return Long.parseLong(value.toString());
- } catch (NumberFormatException e) {
- throw new FilterIllegalMemberException(e);
- }
- case SHORT:
- return ((Short) value).longValue();
- case INT:
- return (Long) value;
- case LONG:
- return (Long) value;
- case DOUBLE:
- return (Long) value;
- case DATE:
- if (value instanceof java.sql.Date) {
- return ((java.sql.Date) value).getTime();
- } else {
- return (Long) value;
- }
- case TIMESTAMP:
- if (value instanceof Timestamp) {
- return ((Timestamp) value).getTime();
- } else {
- return (Long) value;
- }
- default:
- throw new FilterIllegalMemberException(
- "Cannot convert" + this.getDataType().name() + " to Long type value");
- }
- } catch (ClassCastException e) {
- throw new FilterIllegalMemberException(
- "Cannot convert" + this.getDataType().name() + " to Long type value");
- }
-
- }
-
- //Add to judge for BigDecimal
- public BigDecimal getDecimal() throws FilterIllegalMemberException {
- if (value == null) {
- return null;
- }
- try {
- switch (this.getDataType()) {
- case STRING:
- try {
- return new BigDecimal(value.toString());
- } catch (NumberFormatException e) {
- throw new FilterIllegalMemberException(e);
- }
- case SHORT:
- return new BigDecimal((short) value);
- case INT:
- return new BigDecimal((int) value);
- case LONG:
- return new BigDecimal((long) value);
- case DOUBLE:
- return new BigDecimal(value.toString());
- case DECIMAL:
- return new BigDecimal(value.toString());
- case DATE:
- if (value instanceof java.sql.Date) {
- return new BigDecimal(((java.sql.Date) value).getTime());
- } else {
- return new BigDecimal((long) value);
- }
- case TIMESTAMP:
- if (value instanceof Timestamp) {
- return new BigDecimal(((Timestamp) value).getTime());
- } else {
- if (isLiteral) {
- return new BigDecimal((long)value/1000);
- }
- return new BigDecimal((long) value);
- }
- default:
- throw new FilterIllegalMemberException(
- "Cannot convert" + this.getDataType().name() + " to Long type value");
- }
- } catch (ClassCastException e) {
- throw new FilterIllegalMemberException(
- "Cannot convert" + this.getDataType().name() + " to Long type value");
- }
-
- }
-
- public Long getTime() throws FilterIllegalMemberException {
- if (value == null) {
- return null;
- }
- try {
- switch (this.getDataType()) {
- case STRING:
- // Currently the query engine layer only supports yyyy-MM-dd HH:mm:ss date format
- // no matter in which format the data is been stored, so while retrieving the direct
- // surrogate value for filter member first it should be converted in date form as per
- // above format and needs to retrieve time stamp.
- SimpleDateFormat parser =
- new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT);
- Date dateToStr;
- try {
- dateToStr = parser.parse(value.toString());
- return dateToStr.getTime();
- } catch (ParseException e) {
- throw new FilterIllegalMemberException(
- "Cannot convert" + this.getDataType().name() + " to Time/Long type value");
- }
- case SHORT:
- return ((Short) value).longValue();
- case INT:
- case LONG:
- return (Long) value;
- case DOUBLE:
- return (Long) value;
- case DATE:
- if (value instanceof java.sql.Date) {
- return ((Date) value).getTime();
- } else {
- return (Long) value;
- }
- case TIMESTAMP:
- if (value instanceof Timestamp) {
- return ((Timestamp) value).getTime();
- } else {
- if (isLiteral) {
- return (Long) value/1000;
- }
- return (Long) value;
- }
- default:
- throw new FilterIllegalMemberException(
- "Cannot convert" + this.getDataType().name() + " to Time/Long type value");
- }
- } catch (ClassCastException e) {
- throw new FilterIllegalMemberException(
- "Cannot convert" + this.getDataType().name() + " to Time/Long type value");
- }
-
- }
-
- public Boolean getBoolean() throws FilterIllegalMemberException {
- if (value == null) {
- return null;
- }
- try {
- switch (this.getDataType()) {
- case STRING:
- try {
- return Boolean.parseBoolean(value.toString());
- } catch (NumberFormatException e) {
- throw new FilterIllegalMemberException(e);
- }
-
- case BOOLEAN:
- return Boolean.parseBoolean(value.toString());
-
- default:
- throw new FilterIllegalMemberException(
- "Cannot convert" + this.getDataType().name() + " to boolean type value");
- }
- } catch (ClassCastException e) {
- throw new FilterIllegalMemberException(
- "Cannot convert" + this.getDataType().name() + " to Boolean type value");
- }
- }
-
- public List<ExpressionResult> getList() {
- if (null == expressionResults) {
- List<ExpressionResult> a = new ArrayList<ExpressionResult>(20);
- a.add(new ExpressionResult(dataType, value, isLiteral));
- return a;
- } else {
- return expressionResults;
- }
- }
-
- public List<String> getListAsString() throws FilterIllegalMemberException {
- List<String> evaluateResultListFinal = new ArrayList<String>(20);
- List<ExpressionResult> evaluateResultList = getList();
- for (ExpressionResult result : evaluateResultList) {
- String resultString = result.getString();
- if (resultString == null) {
- evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
- continue;
- }
- evaluateResultListFinal.add(resultString);
- }
- return evaluateResultListFinal;
- }
-
- @Override public int hashCode() {
- final int prime = 31;
- int result = 1;
- if (null != expressionResults) {
- result = prime * result + expressionResults.hashCode();
- } else if (null != value) {
- result = prime * result + value.toString().hashCode();
- } else {
- result = prime * result + "".hashCode();
- }
-
- return result;
- }
-
- @Override public boolean equals(Object obj) {
- if (!(obj instanceof ExpressionResult)) {
- return false;
- }
- if (this == obj) {
- return true;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- ExpressionResult objToCompare = (ExpressionResult) obj;
- boolean result = false;
- if (this.value == objToCompare.value) {
- return true;
- }
- try {
- switch (this.getDataType()) {
- case STRING:
- result = this.getString().equals(objToCompare.getString());
- break;
- case SHORT:
- result = this.getShort().equals(objToCompare.getShort());
- break;
- case INT:
- result = this.getInt().equals(objToCompare.getInt());
- break;
- case LONG:
- case DATE:
- case TIMESTAMP:
- result = this.getLong().equals(objToCompare.getLong());
- break;
- case DOUBLE:
- result = this.getDouble().equals(objToCompare.getDouble());
- break;
- case DECIMAL:
- result = this.getDecimal().equals(objToCompare.getDecimal());
- break;
- default:
- break;
- }
- } catch (FilterIllegalMemberException ex) {
- return false;
- }
-
- return result;
- }
-
- public boolean isNull() {
- return value == null;
- }
-
- @Override public int compareTo(ExpressionResult o) {
- try {
- switch (o.dataType) {
- case SHORT:
- case INT:
- case LONG:
- case DOUBLE:
- Double d1 = this.getDouble();
- Double d2 = o.getDouble();
- return d1.compareTo(d2);
- case DECIMAL:
- java.math.BigDecimal val1 = this.getDecimal();
- java.math.BigDecimal val2 = o.getDecimal();
- return val1.compareTo(val2);
- case DATE:
- case TIMESTAMP:
- SimpleDateFormat parser = new SimpleDateFormat(CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
- CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
- Date date1 = null;
- Date date2 = null;
- date1 = parser.parse(this.getString());
- date2 = parser.parse(o.getString());
- return date1.compareTo(date2);
- case STRING:
- default:
- return this.getString().compareTo(o.getString());
- }
- } catch (Exception e) {
- return -1;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/expression/LeafExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/LeafExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/LeafExpression.java
deleted file mode 100644
index 25c9ae6..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/expression/LeafExpression.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression;
-
-public abstract class LeafExpression extends Expression {
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/expression/LiteralExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/LiteralExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/LiteralExpression.java
deleted file mode 100644
index a26cd5d..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/expression/LiteralExpression.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.scan.filter.intf.ExpressionType;
-import org.apache.carbondata.scan.filter.intf.RowIntf;
-
-public class LiteralExpression extends LeafExpression {
-
- /**
- *
- */
- private static final long serialVersionUID = 1L;
- private Object value;
- private DataType dataType;
-
- public LiteralExpression(Object value, DataType dataType) {
- this.value = value;
- this.dataType = dataType;
- }
-
- @Override public ExpressionResult evaluate(RowIntf value) {
- ExpressionResult expressionResult = new ExpressionResult(dataType, this.value, true);
- return expressionResult;
- }
-
- public ExpressionResult getExpressionResult() {
- ExpressionResult expressionResult = new ExpressionResult(dataType, this.value, true);
- return expressionResult;
- }
-
- @Override public ExpressionType getFilterExpressionType() {
- // TODO Auto-generated method stub
- return ExpressionType.LITERAL;
- }
-
- @Override public String getString() {
- // TODO Auto-generated method stub
- return "LiteralExpression(" + value + ')';
- }
-
- /**
- * getLiteralExpDataType.
- *
- * @return
- */
- public DataType getLiteralExpDataType() {
- return dataType;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/expression/UnknownExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/UnknownExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/UnknownExpression.java
deleted file mode 100644
index 01d813d..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/expression/UnknownExpression.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression;
-
-import java.util.List;
-
-public abstract class UnknownExpression extends Expression {
-
- public abstract List<ColumnExpression> getAllColumnList();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/expression/conditional/BinaryConditionalExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/BinaryConditionalExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/conditional/BinaryConditionalExpression.java
deleted file mode 100644
index 9db6953..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/BinaryConditionalExpression.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression.conditional;
-
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.logical.BinaryLogicalExpression;
-
-public abstract class BinaryConditionalExpression extends BinaryLogicalExpression
- implements ConditionalExpression {
-
- /**
- *
- */
- private static final long serialVersionUID = 1L;
- public boolean isNull;
- public BinaryConditionalExpression(Expression left, Expression right) {
- super(left, right);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/expression/conditional/ConditionalExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/ConditionalExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/conditional/ConditionalExpression.java
deleted file mode 100644
index 19653b6..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/ConditionalExpression.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression.conditional;
-
-import java.util.List;
-
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-
-public interface ConditionalExpression {
-
- // Will get the column informations involved in the expressions by
- // traversing the tree
- List<ColumnExpression> getColumnList();
-
- boolean isSingleDimension();
-
- List<ExpressionResult> getLiterals();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/expression/conditional/EqualToExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/EqualToExpression.java b/core/src/main/java/org/apache/carbondata/scan/expression/conditional/EqualToExpression.java
deleted file mode 100644
index db8b739..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/expression/conditional/EqualToExpression.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression.conditional;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.FilterUtil;
-import org.apache.carbondata.scan.filter.intf.ExpressionType;
-import org.apache.carbondata.scan.filter.intf.RowIntf;
-
-public class EqualToExpression extends BinaryConditionalExpression {
-
- private static final long serialVersionUID = 1L;
-
- public EqualToExpression(Expression left, Expression right) {
- super(left, right);
- }
-
- public EqualToExpression(Expression left, Expression right, boolean isNull) {
- super(left, right);
- this.isNull = isNull;
- }
-
- @Override public ExpressionResult evaluate(RowIntf value)
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ExpressionResult elRes = left.evaluate(value);
- ExpressionResult erRes = right.evaluate(value);
-
- boolean result = false;
-
- ExpressionResult val1 = elRes;
- ExpressionResult val2 = erRes;
-
- if (elRes.isNull() || erRes.isNull()) {
- if (isNull) {
- elRes.set(DataType.BOOLEAN, elRes.isNull() == erRes.isNull());
- } else {
- elRes.set(DataType.BOOLEAN, false);
- }
- return elRes;
- }
- //default implementation if the data types are different for the resultsets
- if (elRes.getDataType() != erRes.getDataType()) {
- if (elRes.getDataType().getPrecedenceOrder() < erRes.getDataType().getPrecedenceOrder()) {
- val2 = elRes;
- val1 = erRes;
- }
- }
-
- switch (val1.getDataType()) {
- case STRING:
- result = val1.getString().equals(val2.getString());
- break;
- case SHORT:
- result = val1.getShort().equals(val2.getShort());
- break;
- case INT:
- result = val1.getInt().equals(val2.getInt());
- break;
- case DOUBLE:
- result = FilterUtil.nanSafeEqualsDoubles(val1.getDouble(), val2.getDouble());
- break;
- case DATE:
- case TIMESTAMP:
- result = val1.getTime().equals(val2.getTime());
- break;
- case LONG:
- result = val1.getLong().equals(val2.getLong());
- break;
- case DECIMAL:
- result = val1.getDecimal().compareTo(val2.getDecimal()) == 0;
- break;
- default:
- throw new FilterUnsupportedException(
- "DataType: " + val1.getDataType() + " not supported for the filter expression");
- }
- val1.set(DataType.BOOLEAN, result);
- return val1;
- }
-
- @Override public ExpressionType getFilterExpressionType() {
- return ExpressionType.EQUALS;
- }
-
- @Override public String getString() {
- return "EqualTo(" + left.getString() + ',' + right.getString() + ')';
- }
-
-}
[25/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java
new file mode 100644
index 0000000..c3a0949
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java
@@ -0,0 +1,545 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additiona l information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression;
+
+import java.math.BigDecimal;
+import java.sql.Timestamp;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+
+public class ExpressionResult implements Comparable<ExpressionResult> {
+
+ protected DataType dataType;
+
+ protected Object value;
+
+ private List<ExpressionResult> expressionResults;
+
+ private boolean isLiteral = false;
+
+ public ExpressionResult(DataType dataType, Object value) {
+ this.dataType = dataType;
+ this.value = value;
+ }
+
+ public ExpressionResult(DataType dataType, Object value, boolean isLiteral) {
+ this(dataType, value);
+ this.isLiteral = isLiteral;
+ }
+
+ public ExpressionResult(List<ExpressionResult> expressionResults) {
+ this.expressionResults = expressionResults;
+ }
+
+ public void set(DataType dataType, Object value) {
+ this.dataType = dataType;
+ this.value = value;
+ this.expressionResults = null;
+ }
+
+ public DataType getDataType() {
+ return dataType;
+ }
+
+ public Integer getInt() throws FilterIllegalMemberException {
+ if (value == null) {
+ return null;
+ }
+ try {
+ switch (this.getDataType()) {
+ case STRING:
+ try {
+ return Integer.parseInt(value.toString());
+ } catch (NumberFormatException e) {
+ throw new FilterIllegalMemberException(e);
+ }
+ case SHORT:
+ return ((Short) value).intValue();
+ case INT:
+ case DOUBLE:
+ if (value instanceof Double) {
+ return ((Double) value).intValue();
+ }
+ if (value instanceof Long) {
+ return ((Long) value).intValue();
+ }
+ return (Integer) value;
+ case DATE:
+ if (value instanceof java.sql.Date) {
+ return (int) (((java.sql.Date) value).getTime());
+ } else {
+ return (Integer) value;
+ }
+ case TIMESTAMP:
+ if (value instanceof Timestamp) {
+ return (int) (((Timestamp) value).getTime());
+ } else {
+ if (isLiteral) {
+ Long l = (Long) value /1000;
+ return l.intValue();
+ }
+ return (Integer) value;
+ }
+ default:
+ throw new FilterIllegalMemberException(
+ "Cannot convert" + this.getDataType().name() + " to integer type value");
+ }
+
+ } catch (ClassCastException e) {
+ throw new FilterIllegalMemberException(
+ "Cannot convert" + this.getDataType().name() + " to Integer type value");
+ }
+ }
+
+ public Short getShort() throws FilterIllegalMemberException {
+ if (value == null) {
+ return null;
+ }
+ try {
+ switch (this.getDataType()) {
+ case STRING:
+ try {
+ return Short.parseShort(value.toString());
+ } catch (NumberFormatException e) {
+ throw new FilterIllegalMemberException(e);
+ }
+ case SHORT:
+ case INT:
+ case DOUBLE:
+
+ if (value instanceof Double) {
+ return ((Double) value).shortValue();
+ } else if (value instanceof Integer) {
+ return ((Integer) value).shortValue();
+ }
+ return (Short) value;
+
+ case DATE:
+
+ if (value instanceof java.sql.Date) {
+ return (short) (((java.sql.Date) value).getTime());
+ } else {
+ return (Short) value;
+ }
+ case TIMESTAMP:
+
+ if (value instanceof Timestamp) {
+ return (short) (((Timestamp) value).getTime());
+ } else {
+ if (isLiteral) {
+ Long l = ((long)value/1000);
+ return l.shortValue();
+ }
+ return (Short) value;
+ }
+
+ default:
+ throw new FilterIllegalMemberException(
+ "Cannot convert" + this.getDataType().name() + " to integer type value");
+ }
+
+ } catch (ClassCastException e) {
+ throw new FilterIllegalMemberException(
+ "Cannot convert" + this.getDataType().name() + " to Integer type value");
+ }
+ }
+
+ public String getString() throws FilterIllegalMemberException {
+ if (value == null) {
+ return null;
+ }
+ try {
+ switch (this.getDataType()) {
+ case DATE:
+ case TIMESTAMP:
+ SimpleDateFormat parser = new SimpleDateFormat(CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
+ if (value instanceof Timestamp) {
+ return parser.format((Timestamp) value);
+ } else if (value instanceof java.sql.Date) {
+ return parser.format((java.sql.Date) value);
+ } else if (value instanceof Long) {
+ if (isLiteral) {
+ return parser.format(new Timestamp((long) value/1000));
+ }
+ return parser.format(new Timestamp((long) value));
+ } else if (value instanceof Integer) {
+ return parser.format(new java.sql.Date((int)value));
+ }
+ return value.toString();
+ default:
+ return value.toString();
+ }
+ } catch (Exception e) {
+ throw new FilterIllegalMemberException(
+ "Cannot convert" + this.getDataType().name() + " to String type value");
+ }
+ }
+
+ public Double getDouble() throws FilterIllegalMemberException {
+ if (value == null) {
+ return null;
+ }
+ try {
+ switch (this.getDataType()) {
+ case STRING:
+ try {
+ return Double.parseDouble(value.toString());
+ } catch (NumberFormatException e) {
+ throw new FilterIllegalMemberException(e);
+ }
+ case SHORT:
+ return ((Short) value).doubleValue();
+ case INT:
+ return ((Integer) value).doubleValue();
+ case LONG:
+ return ((Long) value).doubleValue();
+ case DOUBLE:
+ return (Double) value;
+ case DATE:
+ if (value instanceof java.sql.Date) {
+ return (double) ((java.sql.Date) value).getTime();
+ } else {
+ return (Double) (value);
+ }
+ case TIMESTAMP:
+ if (value instanceof Timestamp) {
+ return (double) ((Timestamp) value).getTime();
+ } else {
+ if (isLiteral) {
+ Long l = (Long) value/1000;
+ return l.doubleValue();
+ }
+ return (Double) (value);
+ }
+ default:
+ throw new FilterIllegalMemberException(
+ "Cannot convert" + this.getDataType().name() + " to double type value");
+ }
+ } catch (ClassCastException e) {
+ throw new FilterIllegalMemberException(
+ "Cannot convert" + this.getDataType().name() + " to Double type value");
+ }
+ }
+
+ public Long getLong() throws FilterIllegalMemberException {
+ if (value == null) {
+ return null;
+ }
+ try {
+ switch (this.getDataType()) {
+ case STRING:
+ try {
+ return Long.parseLong(value.toString());
+ } catch (NumberFormatException e) {
+ throw new FilterIllegalMemberException(e);
+ }
+ case SHORT:
+ return ((Short) value).longValue();
+ case INT:
+ return (Long) value;
+ case LONG:
+ return (Long) value;
+ case DOUBLE:
+ return (Long) value;
+ case DATE:
+ if (value instanceof java.sql.Date) {
+ return ((java.sql.Date) value).getTime();
+ } else {
+ return (Long) value;
+ }
+ case TIMESTAMP:
+ if (value instanceof Timestamp) {
+ return ((Timestamp) value).getTime();
+ } else {
+ return (Long) value;
+ }
+ default:
+ throw new FilterIllegalMemberException(
+ "Cannot convert" + this.getDataType().name() + " to Long type value");
+ }
+ } catch (ClassCastException e) {
+ throw new FilterIllegalMemberException(
+ "Cannot convert" + this.getDataType().name() + " to Long type value");
+ }
+
+ }
+
+ //Add to judge for BigDecimal
+ public BigDecimal getDecimal() throws FilterIllegalMemberException {
+ if (value == null) {
+ return null;
+ }
+ try {
+ switch (this.getDataType()) {
+ case STRING:
+ try {
+ return new BigDecimal(value.toString());
+ } catch (NumberFormatException e) {
+ throw new FilterIllegalMemberException(e);
+ }
+ case SHORT:
+ return new BigDecimal((short) value);
+ case INT:
+ return new BigDecimal((int) value);
+ case LONG:
+ return new BigDecimal((long) value);
+ case DOUBLE:
+ return new BigDecimal(value.toString());
+ case DECIMAL:
+ return new BigDecimal(value.toString());
+ case DATE:
+ if (value instanceof java.sql.Date) {
+ return new BigDecimal(((java.sql.Date) value).getTime());
+ } else {
+ return new BigDecimal((long) value);
+ }
+ case TIMESTAMP:
+ if (value instanceof Timestamp) {
+ return new BigDecimal(((Timestamp) value).getTime());
+ } else {
+ if (isLiteral) {
+ return new BigDecimal((long)value/1000);
+ }
+ return new BigDecimal((long) value);
+ }
+ default:
+ throw new FilterIllegalMemberException(
+ "Cannot convert" + this.getDataType().name() + " to Long type value");
+ }
+ } catch (ClassCastException e) {
+ throw new FilterIllegalMemberException(
+ "Cannot convert" + this.getDataType().name() + " to Long type value");
+ }
+
+ }
+
+ public Long getTime() throws FilterIllegalMemberException {
+ if (value == null) {
+ return null;
+ }
+ try {
+ switch (this.getDataType()) {
+ case STRING:
+ // Currently the query engine layer only supports yyyy-MM-dd HH:mm:ss date format
+ // no matter in which format the data is been stored, so while retrieving the direct
+ // surrogate value for filter member first it should be converted in date form as per
+ // above format and needs to retrieve time stamp.
+ SimpleDateFormat parser =
+ new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT);
+ Date dateToStr;
+ try {
+ dateToStr = parser.parse(value.toString());
+ return dateToStr.getTime();
+ } catch (ParseException e) {
+ throw new FilterIllegalMemberException(
+ "Cannot convert" + this.getDataType().name() + " to Time/Long type value");
+ }
+ case SHORT:
+ return ((Short) value).longValue();
+ case INT:
+ case LONG:
+ return (Long) value;
+ case DOUBLE:
+ return (Long) value;
+ case DATE:
+ if (value instanceof java.sql.Date) {
+ return ((Date) value).getTime();
+ } else {
+ return (Long) value;
+ }
+ case TIMESTAMP:
+ if (value instanceof Timestamp) {
+ return ((Timestamp) value).getTime();
+ } else {
+ if (isLiteral) {
+ return (Long) value/1000;
+ }
+ return (Long) value;
+ }
+ default:
+ throw new FilterIllegalMemberException(
+ "Cannot convert" + this.getDataType().name() + " to Time/Long type value");
+ }
+ } catch (ClassCastException e) {
+ throw new FilterIllegalMemberException(
+ "Cannot convert" + this.getDataType().name() + " to Time/Long type value");
+ }
+
+ }
+
+ public Boolean getBoolean() throws FilterIllegalMemberException {
+ if (value == null) {
+ return null;
+ }
+ try {
+ switch (this.getDataType()) {
+ case STRING:
+ try {
+ return Boolean.parseBoolean(value.toString());
+ } catch (NumberFormatException e) {
+ throw new FilterIllegalMemberException(e);
+ }
+
+ case BOOLEAN:
+ return Boolean.parseBoolean(value.toString());
+
+ default:
+ throw new FilterIllegalMemberException(
+ "Cannot convert" + this.getDataType().name() + " to boolean type value");
+ }
+ } catch (ClassCastException e) {
+ throw new FilterIllegalMemberException(
+ "Cannot convert" + this.getDataType().name() + " to Boolean type value");
+ }
+ }
+
+ public List<ExpressionResult> getList() {
+ if (null == expressionResults) {
+ List<ExpressionResult> a = new ArrayList<ExpressionResult>(20);
+ a.add(new ExpressionResult(dataType, value, isLiteral));
+ return a;
+ } else {
+ return expressionResults;
+ }
+ }
+
+ public List<String> getListAsString() throws FilterIllegalMemberException {
+ List<String> evaluateResultListFinal = new ArrayList<String>(20);
+ List<ExpressionResult> evaluateResultList = getList();
+ for (ExpressionResult result : evaluateResultList) {
+ String resultString = result.getString();
+ if (resultString == null) {
+ evaluateResultListFinal.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL);
+ continue;
+ }
+ evaluateResultListFinal.add(resultString);
+ }
+ return evaluateResultListFinal;
+ }
+
+ @Override public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ if (null != expressionResults) {
+ result = prime * result + expressionResults.hashCode();
+ } else if (null != value) {
+ result = prime * result + value.toString().hashCode();
+ } else {
+ result = prime * result + "".hashCode();
+ }
+
+ return result;
+ }
+
+ @Override public boolean equals(Object obj) {
+ if (!(obj instanceof ExpressionResult)) {
+ return false;
+ }
+ if (this == obj) {
+ return true;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ ExpressionResult objToCompare = (ExpressionResult) obj;
+ boolean result = false;
+ if (this.value == objToCompare.value) {
+ return true;
+ }
+ try {
+ switch (this.getDataType()) {
+ case STRING:
+ result = this.getString().equals(objToCompare.getString());
+ break;
+ case SHORT:
+ result = this.getShort().equals(objToCompare.getShort());
+ break;
+ case INT:
+ result = this.getInt().equals(objToCompare.getInt());
+ break;
+ case LONG:
+ case DATE:
+ case TIMESTAMP:
+ result = this.getLong().equals(objToCompare.getLong());
+ break;
+ case DOUBLE:
+ result = this.getDouble().equals(objToCompare.getDouble());
+ break;
+ case DECIMAL:
+ result = this.getDecimal().equals(objToCompare.getDecimal());
+ break;
+ default:
+ break;
+ }
+ } catch (FilterIllegalMemberException ex) {
+ return false;
+ }
+
+ return result;
+ }
+
+ public boolean isNull() {
+ return value == null;
+ }
+
+ @Override public int compareTo(ExpressionResult o) {
+ try {
+ switch (o.dataType) {
+ case SHORT:
+ case INT:
+ case LONG:
+ case DOUBLE:
+ Double d1 = this.getDouble();
+ Double d2 = o.getDouble();
+ return d1.compareTo(d2);
+ case DECIMAL:
+ java.math.BigDecimal val1 = this.getDecimal();
+ java.math.BigDecimal val2 = o.getDecimal();
+ return val1.compareTo(val2);
+ case DATE:
+ case TIMESTAMP:
+ SimpleDateFormat parser = new SimpleDateFormat(CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT,
+ CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
+ Date date1 = null;
+ Date date2 = null;
+ date1 = parser.parse(this.getString());
+ date2 = parser.parse(o.getString());
+ return date1.compareTo(date2);
+ case STRING:
+ default:
+ return this.getString().compareTo(o.getString());
+ }
+ } catch (Exception e) {
+ return -1;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/expression/LeafExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/LeafExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/LeafExpression.java
new file mode 100644
index 0000000..6c79b08
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/LeafExpression.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression;
+
+public abstract class LeafExpression extends Expression {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/expression/LiteralExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/LiteralExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/LiteralExpression.java
new file mode 100644
index 0000000..73b2114
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/LiteralExpression.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+
+public class LiteralExpression extends LeafExpression {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+ private Object value;
+ private DataType dataType;
+
+ public LiteralExpression(Object value, DataType dataType) {
+ this.value = value;
+ this.dataType = dataType;
+ }
+
+ @Override public ExpressionResult evaluate(RowIntf value) {
+ ExpressionResult expressionResult = new ExpressionResult(dataType, this.value, true);
+ return expressionResult;
+ }
+
+ public ExpressionResult getExpressionResult() {
+ ExpressionResult expressionResult = new ExpressionResult(dataType, this.value, true);
+ return expressionResult;
+ }
+
+ @Override public ExpressionType getFilterExpressionType() {
+ // TODO Auto-generated method stub
+ return ExpressionType.LITERAL;
+ }
+
+ @Override public String getString() {
+ // TODO Auto-generated method stub
+ return "LiteralExpression(" + value + ')';
+ }
+
+ /**
+ * getLiteralExpDataType.
+ *
+ * @return
+ */
+ public DataType getLiteralExpDataType() {
+ return dataType;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/expression/UnknownExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/UnknownExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/UnknownExpression.java
new file mode 100644
index 0000000..818720a
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/UnknownExpression.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression;
+
+import java.util.List;
+
+public abstract class UnknownExpression extends Expression {
+
+ public abstract List<ColumnExpression> getAllColumnList();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/BinaryConditionalExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/BinaryConditionalExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/BinaryConditionalExpression.java
new file mode 100644
index 0000000..f6725de
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/BinaryConditionalExpression.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.conditional;
+
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.logical.BinaryLogicalExpression;
+
+public abstract class BinaryConditionalExpression extends BinaryLogicalExpression
+ implements ConditionalExpression {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+ public boolean isNull;
+ public BinaryConditionalExpression(Expression left, Expression right) {
+ super(left, right);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ConditionalExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ConditionalExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ConditionalExpression.java
new file mode 100644
index 0000000..6f6e7fa
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ConditionalExpression.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.conditional;
+
+import java.util.List;
+
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+
+public interface ConditionalExpression {
+
+ // Will get the column informations involved in the expressions by
+ // traversing the tree
+ List<ColumnExpression> getColumnList();
+
+ boolean isSingleDimension();
+
+ List<ExpressionResult> getLiterals();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpression.java
new file mode 100644
index 0000000..45dc33f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/EqualToExpression.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.conditional;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+
+public class EqualToExpression extends BinaryConditionalExpression {
+
+ private static final long serialVersionUID = 1L;
+
+ public EqualToExpression(Expression left, Expression right) {
+ super(left, right);
+ }
+
+ public EqualToExpression(Expression left, Expression right, boolean isNull) {
+ super(left, right);
+ this.isNull = isNull;
+ }
+
+ @Override public ExpressionResult evaluate(RowIntf value)
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ExpressionResult elRes = left.evaluate(value);
+ ExpressionResult erRes = right.evaluate(value);
+
+ boolean result = false;
+
+ ExpressionResult val1 = elRes;
+ ExpressionResult val2 = erRes;
+
+ if (elRes.isNull() || erRes.isNull()) {
+ if (isNull) {
+ elRes.set(DataType.BOOLEAN, elRes.isNull() == erRes.isNull());
+ } else {
+ elRes.set(DataType.BOOLEAN, false);
+ }
+ return elRes;
+ }
+ //default implementation if the data types are different for the resultsets
+ if (elRes.getDataType() != erRes.getDataType()) {
+ if (elRes.getDataType().getPrecedenceOrder() < erRes.getDataType().getPrecedenceOrder()) {
+ val2 = elRes;
+ val1 = erRes;
+ }
+ }
+
+ switch (val1.getDataType()) {
+ case STRING:
+ result = val1.getString().equals(val2.getString());
+ break;
+ case SHORT:
+ result = val1.getShort().equals(val2.getShort());
+ break;
+ case INT:
+ result = val1.getInt().equals(val2.getInt());
+ break;
+ case DOUBLE:
+ result = FilterUtil.nanSafeEqualsDoubles(val1.getDouble(), val2.getDouble());
+ break;
+ case DATE:
+ case TIMESTAMP:
+ result = val1.getTime().equals(val2.getTime());
+ break;
+ case LONG:
+ result = val1.getLong().equals(val2.getLong());
+ break;
+ case DECIMAL:
+ result = val1.getDecimal().compareTo(val2.getDecimal()) == 0;
+ break;
+ default:
+ throw new FilterUnsupportedException(
+ "DataType: " + val1.getDataType() + " not supported for the filter expression");
+ }
+ val1.set(DataType.BOOLEAN, result);
+ return val1;
+ }
+
+ @Override public ExpressionType getFilterExpressionType() {
+ return ExpressionType.EQUALS;
+ }
+
+ @Override public String getString() {
+ return "EqualTo(" + left.getString() + ',' + right.getString() + ')';
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpression.java
new file mode 100644
index 0000000..b572fe2
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanEqualToExpression.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.conditional;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+
+public class GreaterThanEqualToExpression extends BinaryConditionalExpression {
+ private static final long serialVersionUID = 4185317066280688984L;
+
+ public GreaterThanEqualToExpression(Expression left, Expression right) {
+ super(left, right);
+ }
+
+ public ExpressionResult evaluate(RowIntf value)
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ExpressionResult elRes = left.evaluate(value);
+ ExpressionResult erRes = right.evaluate(value);
+ ExpressionResult exprResVal1 = elRes;
+ if (elRes.isNull() || erRes.isNull()) {
+ elRes.set(DataType.BOOLEAN, false);
+ return elRes;
+ }
+ if (elRes.getDataType() != erRes.getDataType()) {
+ if (elRes.getDataType().getPrecedenceOrder() < erRes.getDataType().getPrecedenceOrder()) {
+ exprResVal1 = erRes;
+ }
+
+ }
+ boolean result = false;
+ switch (exprResVal1.getDataType()) {
+ case STRING:
+ result = elRes.getString().compareTo(erRes.getString()) >= 0;
+ break;
+ case SHORT:
+ result = elRes.getShort() >= (erRes.getShort());
+ break;
+ case INT:
+ result = elRes.getInt() >= (erRes.getInt());
+ break;
+ case DOUBLE:
+ result = elRes.getDouble() >= (erRes.getDouble());
+ break;
+ case DATE:
+ case TIMESTAMP:
+ result = elRes.getTime() >= (erRes.getTime());
+ break;
+ case LONG:
+ result = elRes.getLong() >= (erRes.getLong());
+ break;
+ case DECIMAL:
+ result = elRes.getDecimal().compareTo(erRes.getDecimal()) >= 0;
+ break;
+ default:
+ throw new FilterUnsupportedException(
+ "DataType: " + exprResVal1.getDataType() + " not supported for the filter expression");
+ }
+ exprResVal1.set(DataType.BOOLEAN, result);
+ return exprResVal1;
+ }
+
+ @Override public ExpressionType getFilterExpressionType() {
+ return ExpressionType.GREATERTHAN_EQUALTO;
+ }
+
+ @Override public String getString() {
+ return "GreaterThanEqualTo(" + left.getString() + ',' + right.getString() + ')';
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpression.java
new file mode 100644
index 0000000..78c9381
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpression.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.conditional;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+
+
+public class GreaterThanExpression extends BinaryConditionalExpression {
+ private static final long serialVersionUID = -5319109756575539219L;
+
+ public GreaterThanExpression(Expression left, Expression right) {
+ super(left, right);
+ }
+
+ @Override public ExpressionResult evaluate(RowIntf value)
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ExpressionResult exprLeftRes = left.evaluate(value);
+ ExpressionResult exprRightRes = right.evaluate(value);
+ ExpressionResult val1 = exprLeftRes;
+ if (exprLeftRes.isNull() || exprRightRes.isNull()) {
+ exprLeftRes.set(DataType.BOOLEAN, false);
+ return exprLeftRes;
+ }
+ if (exprLeftRes.getDataType() != exprRightRes.getDataType()) {
+ if (exprLeftRes.getDataType().getPrecedenceOrder() < exprRightRes.getDataType()
+ .getPrecedenceOrder()) {
+ val1 = exprRightRes;
+ }
+
+ }
+ boolean result = false;
+ switch (val1.getDataType()) {
+ case STRING:
+ result = exprLeftRes.getString().compareTo(exprRightRes.getString()) > 0;
+ break;
+ case DOUBLE:
+ result = exprLeftRes.getDouble() > (exprRightRes.getDouble());
+ break;
+ case SHORT:
+ result = exprLeftRes.getShort() > (exprRightRes.getShort());
+ break;
+ case INT:
+ result = exprLeftRes.getInt() > (exprRightRes.getInt());
+ break;
+ case DATE:
+ case TIMESTAMP:
+ result = exprLeftRes.getTime() > (exprRightRes.getTime());
+ break;
+ case LONG:
+ result = exprLeftRes.getLong() > (exprRightRes.getLong());
+ break;
+ case DECIMAL:
+ result = exprLeftRes.getDecimal().compareTo(exprRightRes.getDecimal()) > 0;
+ break;
+ default:
+ throw new FilterUnsupportedException(
+ "DataType: " + val1.getDataType() + " not supported for the filter expression");
+ }
+ val1.set(DataType.BOOLEAN, result);
+ return val1;
+ }
+
+ @Override public ExpressionType getFilterExpressionType() {
+ return ExpressionType.GREATERTHAN;
+ }
+
+ @Override public String getString() {
+ return "GreaterThan(" + left.getString() + ',' + right.getString() + ')';
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/InExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/InExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/InExpression.java
new file mode 100644
index 0000000..4ae1c04
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/InExpression.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.conditional;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+
+public class InExpression extends BinaryConditionalExpression {
+ private static final long serialVersionUID = -3149927446694175489L;
+
+ protected transient Set<ExpressionResult> setOfExprResult;
+
+ public InExpression(Expression left, Expression right) {
+ super(left, right);
+ }
+
+ @Override public ExpressionResult evaluate(RowIntf value)
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ExpressionResult leftRsult = left.evaluate(value);
+
+ if (setOfExprResult == null) {
+ ExpressionResult rightRsult = right.evaluate(value);
+ ExpressionResult val = null;
+ setOfExprResult = new HashSet<ExpressionResult>(10);
+ for (ExpressionResult expressionResVal : rightRsult.getList()) {
+ if (expressionResVal.getDataType().getPrecedenceOrder() < leftRsult.getDataType()
+ .getPrecedenceOrder()) {
+ val = leftRsult;
+ } else {
+ val = expressionResVal;
+ }
+ switch (val.getDataType()) {
+ case STRING:
+ val = new ExpressionResult(val.getDataType(), expressionResVal.getString());
+ break;
+ case SHORT:
+ val = new ExpressionResult(val.getDataType(), expressionResVal.getShort());
+ break;
+ case INT:
+ val = new ExpressionResult(val.getDataType(), expressionResVal.getInt());
+ break;
+ case DOUBLE:
+ val = new ExpressionResult(val.getDataType(), expressionResVal.getDouble());
+ break;
+ case LONG:
+ val = new ExpressionResult(val.getDataType(), expressionResVal.getLong());
+ break;
+ case DATE:
+ case TIMESTAMP:
+ val = new ExpressionResult(val.getDataType(), expressionResVal.getTime());
+ break;
+ case DECIMAL:
+ val = new ExpressionResult(val.getDataType(), expressionResVal.getDecimal());
+ break;
+ default:
+ throw new FilterUnsupportedException(
+ "DataType: " + val.getDataType() + " not supported for the filter expression");
+ }
+ setOfExprResult.add(val);
+ }
+ }
+ leftRsult.set(DataType.BOOLEAN, setOfExprResult.contains(leftRsult));
+ return leftRsult;
+ }
+
+ @Override public ExpressionType getFilterExpressionType() {
+ return ExpressionType.IN;
+ }
+
+ @Override public String getString() {
+ return "IN(" + left.getString() + ',' + right.getString() + ')';
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpression.java
new file mode 100644
index 0000000..688ed5d
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpression.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.conditional;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+
+public class LessThanEqualToExpression extends BinaryConditionalExpression {
+ private static final long serialVersionUID = 1L;
+
+ public LessThanEqualToExpression(Expression left, Expression right) {
+ super(left, right);
+ }
+
+ public ExpressionResult evaluate(RowIntf value)
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ExpressionResult elRes = left.evaluate(value);
+ ExpressionResult erRes = right.evaluate(value);
+ ExpressionResult exprResValue1 = elRes;
+ if (elRes.isNull() || erRes.isNull()) {
+ elRes.set(DataType.BOOLEAN, false);
+ return elRes;
+ }
+ if (elRes.getDataType() != erRes.getDataType()) {
+ if (elRes.getDataType().getPrecedenceOrder() < erRes.getDataType().getPrecedenceOrder()) {
+ exprResValue1 = erRes;
+ }
+
+ }
+ boolean result = false;
+ switch (exprResValue1.getDataType()) {
+ case STRING:
+ result = elRes.getString().compareTo(erRes.getString()) <= 0;
+ break;
+ case SHORT:
+ result = elRes.getShort() <= (erRes.getShort());
+ break;
+ case INT:
+ result = elRes.getInt() <= (erRes.getInt());
+ break;
+ case DOUBLE:
+ result = elRes.getDouble() <= (erRes.getDouble());
+ break;
+ case DATE:
+ case TIMESTAMP:
+ result = elRes.getTime() <= (erRes.getTime());
+ break;
+ case LONG:
+ result = elRes.getLong() <= (erRes.getLong());
+ break;
+ case DECIMAL:
+ result = elRes.getDecimal().compareTo(erRes.getDecimal()) <= 0;
+ break;
+ default:
+ throw new FilterUnsupportedException("DataType: " + exprResValue1.getDataType()
+ + " not supported for the filter expression");
+ }
+ exprResValue1.set(DataType.BOOLEAN, result);
+ return exprResValue1;
+ }
+
+ @Override public ExpressionType getFilterExpressionType() {
+ // TODO Auto-generated method stub
+ return ExpressionType.LESSTHAN_EQUALTO;
+ }
+
+ @Override public String getString() {
+ return "LessThanEqualTo(" + left.getString() + ',' + right.getString() + ')';
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpression.java
new file mode 100644
index 0000000..a575494
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpression.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.conditional;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+
+public class LessThanExpression extends BinaryConditionalExpression {
+
+ private static final long serialVersionUID = 6343040416663699924L;
+
+ public LessThanExpression(Expression left, Expression right) {
+ super(left, right);
+ }
+
+ public ExpressionResult evaluate(RowIntf value)
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ExpressionResult erRes = right.evaluate(value);
+ ExpressionResult elRes = left.evaluate(value);
+
+ ExpressionResult val1 = elRes;
+
+ boolean result = false;
+
+ if (elRes.isNull() || erRes.isNull()) {
+ elRes.set(DataType.BOOLEAN, false);
+ return elRes;
+ }
+ if (elRes.getDataType() != erRes.getDataType()) {
+ if (elRes.getDataType().getPrecedenceOrder() < erRes.getDataType().getPrecedenceOrder()) {
+ val1 = erRes;
+ }
+
+ }
+ switch (val1.getDataType()) {
+ case STRING:
+ result = elRes.getString().compareTo(erRes.getString()) < 0;
+ break;
+ case SHORT:
+ result = elRes.getShort() < (erRes.getShort());
+ break;
+ case INT:
+ result = elRes.getInt() < (erRes.getInt());
+ break;
+ case DOUBLE:
+ result = elRes.getDouble() < (erRes.getDouble());
+ break;
+ case DATE:
+ case TIMESTAMP:
+ result = elRes.getTime() < (erRes.getTime());
+ break;
+ case LONG:
+ result = elRes.getLong() < (erRes.getLong());
+ break;
+ case DECIMAL:
+ result = elRes.getDecimal().compareTo(erRes.getDecimal()) < 0;
+ break;
+ default:
+ throw new FilterUnsupportedException(
+ "DataType: " + val1.getDataType() + " not supported for the filter expression");
+ }
+ val1.set(DataType.BOOLEAN, result);
+ return val1;
+ }
+
+ @Override public ExpressionType getFilterExpressionType() {
+ return ExpressionType.LESSTHAN;
+ }
+
+ @Override public String getString() {
+ return "LessThan(" + left.getString() + ',' + right.getString() + ')';
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ListExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ListExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ListExpression.java
new file mode 100644
index 0000000..831437a
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/ListExpression.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.conditional;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+
+public class ListExpression extends Expression {
+ private static final long serialVersionUID = 1L;
+
+ public ListExpression(List<Expression> children) {
+ this.children = children;
+ }
+
+ @Override public ExpressionResult evaluate(RowIntf value) throws FilterUnsupportedException {
+ List<ExpressionResult> listOfExprRes = new ArrayList<ExpressionResult>(10);
+
+ for (Expression expr : children) {
+ try {
+ listOfExprRes.add(expr.evaluate(value));
+ } catch (FilterIllegalMemberException e) {
+ continue;
+ }
+ }
+ return new ExpressionResult(listOfExprRes);
+ }
+
+ @Override public ExpressionType getFilterExpressionType() {
+ // TODO Auto-generated method stub
+ return ExpressionType.LIST;
+ }
+
+ @Override public String getString() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpression.java
new file mode 100644
index 0000000..2f656cd
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpression.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.conditional;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+
+public class NotEqualsExpression extends BinaryConditionalExpression {
+
+ private static final long serialVersionUID = 8684006025540863973L;
+ private boolean isNotNull = false;
+ public NotEqualsExpression(Expression left, Expression right, boolean isNotNull) {
+ super(left, right);
+ this.isNotNull = isNotNull;
+ }
+
+ public NotEqualsExpression(Expression left, Expression right) {
+ super(left, right);
+ }
+
+ @Override public ExpressionResult evaluate(RowIntf value)
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ExpressionResult elRes = left.evaluate(value);
+ ExpressionResult erRes = right.evaluate(value);
+
+ boolean result = false;
+ ExpressionResult val1 = elRes;
+ ExpressionResult val2 = erRes;
+ if (elRes.isNull() || erRes.isNull()) {
+ if (isNotNull) {
+ elRes.set(DataType.BOOLEAN, elRes.isNull() != erRes.isNull());
+ } else {
+ elRes.set(DataType.BOOLEAN, false);
+ }
+ return elRes;
+ }
+ //default implementation if the data types are different for the resultsets
+ if (elRes.getDataType() != erRes.getDataType()) {
+ // result = elRes.getString().equals(erRes.getString());
+ if (elRes.getDataType().getPrecedenceOrder() < erRes.getDataType().getPrecedenceOrder()) {
+ val1 = erRes;
+ val2 = elRes;
+ }
+ }
+ switch (val1.getDataType()) {
+ case STRING:
+ result = !val1.getString().equals(val2.getString());
+ break;
+ case SHORT:
+ result = val1.getShort().shortValue() != val2.getShort().shortValue();
+ break;
+ case INT:
+ result = val1.getInt().intValue() != val2.getInt().intValue();
+ break;
+ case DOUBLE:
+ result = val1.getDouble().doubleValue() != val2.getDouble().doubleValue();
+ break;
+ case DATE:
+ case TIMESTAMP:
+ result = val1.getTime().longValue() != val2.getTime().longValue();
+ break;
+ case LONG:
+ result = elRes.getLong().longValue() != (erRes.getLong()).longValue();
+ break;
+ case DECIMAL:
+ result = elRes.getDecimal().compareTo(erRes.getDecimal()) != 0;
+ break;
+ default:
+ throw new FilterUnsupportedException(
+ "DataType: " + val1.getDataType() + " not supported for the filter expression");
+ }
+ val1.set(DataType.BOOLEAN, result);
+ return val1;
+ }
+
+ @Override public ExpressionType getFilterExpressionType() {
+ return ExpressionType.NOT_EQUALS;
+ }
+
+ @Override public String getString() {
+ return "NotEquals(" + left.getString() + ',' + right.getString() + ')';
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpression.java
new file mode 100644
index 0000000..e56d6f6
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpression.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.conditional;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+
+public class NotInExpression extends BinaryConditionalExpression {
+ private static final long serialVersionUID = -6835841923752118034L;
+ protected transient Set<ExpressionResult> setOfExprResult;
+
+ public NotInExpression(Expression left, Expression right) {
+ super(left, right);
+ }
+
+ @Override public ExpressionResult evaluate(RowIntf value)
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ExpressionResult leftRsult = left.evaluate(value);
+ if (setOfExprResult == null) {
+ ExpressionResult val = null;
+ ExpressionResult rightRsult = right.evaluate(value);
+ setOfExprResult = new HashSet<ExpressionResult>(10);
+ for (ExpressionResult exprResVal : rightRsult.getList()) {
+ if (exprResVal.getDataType().getPrecedenceOrder() < leftRsult.getDataType()
+ .getPrecedenceOrder()) {
+ val = leftRsult;
+ } else {
+ val = exprResVal;
+ }
+ switch (val.getDataType()) {
+ case STRING:
+ val = new ExpressionResult(val.getDataType(), exprResVal.getString());
+ break;
+ case SHORT:
+ val = new ExpressionResult(val.getDataType(), exprResVal.getShort());
+ break;
+ case INT:
+ val = new ExpressionResult(val.getDataType(), exprResVal.getInt());
+ break;
+ case DOUBLE:
+ val = new ExpressionResult(val.getDataType(), exprResVal.getDouble());
+ break;
+ case DATE:
+ case TIMESTAMP:
+ val = new ExpressionResult(val.getDataType(), exprResVal.getTime());
+ break;
+ case LONG:
+ val = new ExpressionResult(val.getDataType(), exprResVal.getLong());
+ break;
+ case DECIMAL:
+ val = new ExpressionResult(val.getDataType(), exprResVal.getDecimal());
+ break;
+ default:
+ throw new FilterUnsupportedException(
+ "DataType: " + val.getDataType() + " not supported for the filter expression");
+ }
+ setOfExprResult.add(val);
+ }
+ }
+ leftRsult.set(DataType.BOOLEAN, !setOfExprResult.contains(leftRsult));
+
+ return leftRsult;
+ }
+
+ @Override public ExpressionType getFilterExpressionType() {
+ return ExpressionType.NOT_IN;
+ }
+
+ @Override public String getString() {
+ return "NOT IN(" + left.getString() + ',' + right.getString() + ')';
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterIllegalMemberException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterIllegalMemberException.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterIllegalMemberException.java
new file mode 100644
index 0000000..80c364f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterIllegalMemberException.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.exception;
+
+import java.util.Locale;
+
+/**
+ * FilterIllegalMemberException class representing exception which can cause while evaluating
+ * filter members needs to be gracefully handled without propagating to outer layer so that
+ * the execution should not get interrupted.
+ */
+public class FilterIllegalMemberException extends Exception {
+
+ /**
+ * default serial version ID.
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * The Error message.
+ */
+ private String msg = "";
+
+ /**
+ * Constructor
+ *
+ * @param msg The error message for this exception.
+ */
+ public FilterIllegalMemberException(String msg) {
+ super(msg);
+ this.msg = msg;
+ }
+
+ /**
+ * Constructor
+ *
+ * @param msg The error message for this exception.
+ */
+ public FilterIllegalMemberException(String msg, Throwable t) {
+ super(msg, t);
+ this.msg = msg;
+ }
+
+ /**
+ * Constructor
+ *
+ */
+ public FilterIllegalMemberException(Throwable t) {
+ super(t);
+ }
+
+ /**
+ * This method is used to get the localized message.
+ *
+ * @param locale - A Locale object represents a specific geographical,
+ * political, or cultural region.
+ * @return - Localized error message.
+ */
+ public String getLocalizedMessage(Locale locale) {
+ return "";
+ }
+
+ /**
+ * getLocalizedMessage
+ */
+ @Override public String getLocalizedMessage() {
+ return super.getLocalizedMessage();
+ }
+
+ /**
+ * getMessage
+ */
+ public String getMessage() {
+ return this.msg;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterUnsupportedException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterUnsupportedException.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterUnsupportedException.java
new file mode 100644
index 0000000..4f88944
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/exception/FilterUnsupportedException.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.exception;
+
+import java.util.Locale;
+
+public class FilterUnsupportedException extends Exception {
+
+ /**
+ * default serial version ID.
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * The Error message.
+ */
+ private String msg = "";
+
+ /**
+ * Constructor
+ *
+ * @param msg The error message for this exception.
+ */
+ public FilterUnsupportedException(String msg) {
+ super(msg);
+ this.msg = msg;
+ }
+
+ /**
+ * Constructor
+ *
+ * @param msg The error message for this exception.
+ */
+ public FilterUnsupportedException(String msg, Throwable t) {
+ super(msg, t);
+ this.msg = msg;
+ }
+
+ /**
+ * Constructor
+ *
+ */
+ public FilterUnsupportedException(Throwable t) {
+ super(t);
+ }
+
+ /**
+ * This method is used to get the localized message.
+ *
+ * @param locale - A Locale object represents a specific geographical,
+ * political, or cultural region.
+ * @return - Localized error message.
+ */
+ public String getLocalizedMessage(Locale locale) {
+ return "";
+ }
+
+ /**
+ * getLocalizedMessage
+ */
+ @Override public String getLocalizedMessage() {
+ return super.getLocalizedMessage();
+ }
+
+ /**
+ * getMessage
+ */
+ public String getMessage() {
+ return this.msg;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/AndExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/AndExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/AndExpression.java
new file mode 100644
index 0000000..bbb1ef2
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/AndExpression.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.logical;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+
+public class AndExpression extends BinaryLogicalExpression {
+
+ private static final long serialVersionUID = 1L;
+
+ public AndExpression(Expression left, Expression right) {
+ super(left, right);
+ }
+
+ @Override public ExpressionResult evaluate(RowIntf value)
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ExpressionResult resultLeft = left.evaluate(value);
+ ExpressionResult resultRight = right.evaluate(value);
+ switch (resultLeft.getDataType()) {
+ case BOOLEAN:
+ resultLeft.set(DataType.BOOLEAN, (resultLeft.getBoolean() && resultRight.getBoolean()));
+ break;
+ default:
+ throw new FilterUnsupportedException(
+ "Incompatible datatype for applying AND Expression Filter");
+ }
+ return resultLeft;
+ }
+
+ @Override public ExpressionType getFilterExpressionType() {
+ // TODO Auto-generated method stub
+ return ExpressionType.AND;
+ }
+
+ @Override public String getString() {
+ // TODO Auto-generated method stub
+ return "And(" + left.getString() + ',' + right.getString() + ')';
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/BinaryLogicalExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/BinaryLogicalExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/BinaryLogicalExpression.java
new file mode 100644
index 0000000..9209959
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/BinaryLogicalExpression.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.logical;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.scan.expression.BinaryExpression;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.LiteralExpression;
+
+public abstract class BinaryLogicalExpression extends BinaryExpression {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ public BinaryLogicalExpression(Expression left, Expression right) {
+ super(left, right);
+ // TODO Auto-generated constructor stub
+ }
+
+ public List<ExpressionResult> getLiterals() {
+ List<ExpressionResult> listOfExp =
+ new ArrayList<ExpressionResult>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ getExpressionResultList(this, listOfExp);
+ Collections.sort(listOfExp);
+ return listOfExp;
+ }
+
+ // Will get the column informations involved in the expressions by
+ // traversing the tree
+ public List<ColumnExpression> getColumnList() {
+ // TODO
+ List<ColumnExpression> listOfExp =
+ new ArrayList<ColumnExpression>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ getColumnList(this, listOfExp);
+ return listOfExp;
+ }
+
+ private void getColumnList(Expression expression, List<ColumnExpression> lst) {
+ if (expression instanceof ColumnExpression) {
+ ColumnExpression colExp = (ColumnExpression) expression;
+ boolean found = false;
+
+ for (ColumnExpression currentColExp : lst) {
+ if (currentColExp.getColumnName().equals(colExp.getColumnName())) {
+ found = true;
+ colExp.setColIndex(currentColExp.getColIndex());
+ break;
+ }
+ }
+ if (!found) {
+ colExp.setColIndex(lst.size());
+ lst.add(colExp);
+ }
+ }
+ for (Expression child : expression.getChildren()) {
+ getColumnList(child, lst);
+ }
+ }
+
+ public boolean isSingleDimension() {
+ List<ColumnExpression> listOfExp =
+ new ArrayList<ColumnExpression>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ getColumnList(this, listOfExp);
+ if (listOfExp.size() == 1 && listOfExp.get(0).isDimension()) {
+ return true;
+ }
+ return false;
+
+ }
+
+ private void getExpressionResultList(Expression binaryConditionalExpression,
+ List<ExpressionResult> listOfExp) {
+ if (binaryConditionalExpression instanceof LiteralExpression) {
+ ExpressionResult colExp =
+ ((LiteralExpression) binaryConditionalExpression).getExpressionResult();
+ listOfExp.add(colExp);
+ }
+ for (Expression child : binaryConditionalExpression.getChildren()) {
+ getExpressionResultList(child, listOfExp);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/FalseExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/FalseExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/FalseExpression.java
new file mode 100644
index 0000000..64a8714
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/FalseExpression.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.logical;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.LiteralExpression;
+import org.apache.carbondata.core.scan.expression.conditional.BinaryConditionalExpression;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+
+
+
+/**
+ * This class will form an expression whose evaluation will be always false.
+ */
+public class FalseExpression extends BinaryConditionalExpression {
+
+
+ private static final long serialVersionUID = -8390184061336799370L;
+
+ public FalseExpression(Expression child1) {
+ super(child1, new LiteralExpression(null,null));
+ }
+
+ /**
+ * This method will always return false, mainly used in the filter expressions
+ * which are illogical.
+ * eg: columnName NOT IN('Java',NULL)
+ * @param value
+ * @return
+ * @throws FilterUnsupportedException
+ * @throws FilterIllegalMemberException
+ */
+ @Override public ExpressionResult evaluate(RowIntf value)
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ return new ExpressionResult(DataType.BOOLEAN,false);
+ }
+
+ /**
+ * This method will return the expression types
+ * @return
+ */
+ @Override public ExpressionType getFilterExpressionType() {
+ return ExpressionType.FALSE;
+ }
+ @Override public String getString() {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/OrExpression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/OrExpression.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/OrExpression.java
new file mode 100644
index 0000000..13de9f4
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/logical/OrExpression.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.logical;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+
+public class OrExpression extends BinaryLogicalExpression {
+
+ private static final long serialVersionUID = 4220598043176438380L;
+
+ public OrExpression(Expression left, Expression right) {
+ super(left, right);
+ }
+
+ @Override public ExpressionResult evaluate(RowIntf value)
+ throws FilterIllegalMemberException, FilterUnsupportedException {
+ ExpressionResult resultLeft = left.evaluate(value);
+ ExpressionResult resultRight = right.evaluate(value);
+ switch (resultLeft.getDataType()) {
+ case BOOLEAN:
+ resultLeft.set(DataType.BOOLEAN, (resultLeft.getBoolean() || resultRight.getBoolean()));
+ break;
+ default:
+ throw new FilterUnsupportedException(
+ "Incompatible datatype for applying OR Expression Filter");
+ }
+
+ return resultLeft;
+ }
+
+ @Override public ExpressionType getFilterExpressionType() {
+ return ExpressionType.OR;
+ }
+
+ @Override public String getString() {
+ return "Or(" + left.getString() + ',' + right.getString() + ')';
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/filter/DimColumnFilterInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/DimColumnFilterInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/DimColumnFilterInfo.java
new file mode 100644
index 0000000..ece7944
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/DimColumnFilterInfo.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.filter;
+
+import java.io.Serializable;
+import java.util.List;
+
+public class DimColumnFilterInfo implements Serializable {
+
+ private static final long serialVersionUID = 8181578747306832771L;
+
+ private boolean isIncludeFilter;
+
+ private List<Integer> filterList;
+
+ /**
+ * Implicit column filter values to be used for block and blocklet pruning
+ */
+ private List<String> implicitColumnFilterList;
+ private List<Integer> excludeFilterList;
+ /**
+ * maintain the no dictionary filter values list.
+ */
+ private List<byte[]> noDictionaryFilterValuesList;
+
+ public List<byte[]> getNoDictionaryFilterValuesList() {
+ return noDictionaryFilterValuesList;
+ }
+
+ public boolean isIncludeFilter() {
+ return isIncludeFilter;
+ }
+
+ public void setIncludeFilter(boolean isIncludeFilter) {
+ this.isIncludeFilter = isIncludeFilter;
+ }
+
+ public List<Integer> getFilterList() {
+ return filterList;
+ }
+
+ public void setFilterList(List<Integer> filterList) {
+ this.filterList = filterList;
+ }
+
+ public void setFilterListForNoDictionaryCols(List<byte[]> noDictionaryFilterValuesList) {
+ this.noDictionaryFilterValuesList = noDictionaryFilterValuesList;
+ }
+
+ public List<Integer> getExcludeFilterList() {
+ return excludeFilterList;
+ }
+ public void setExcludeFilterList(List<Integer> excludeFilterList) {
+ this.excludeFilterList = excludeFilterList;
+ }
+ public List<String> getImplicitColumnFilterList() {
+ return implicitColumnFilterList;
+ }
+
+ public void setImplicitColumnFilterList(List<String> implicitColumnFilterList) {
+ this.implicitColumnFilterList = implicitColumnFilterList;
+ }
+}
[27/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
new file mode 100644
index 0000000..c62def7
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.collector.impl;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.carbondata.core.update.data.BlockletLevelDeleteDeltaDataCache;
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.metadata.Encoding;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.filter.GenericQueryType;
+import org.apache.carbondata.core.scan.model.QueryDimension;
+import org.apache.carbondata.core.scan.model.QueryMeasure;
+import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+
+import org.apache.commons.lang3.ArrayUtils;
+/**
+ * It is not a collector it is just a scanned result holder.
+ */
+public class DictionaryBasedResultCollector extends AbstractScannedResultCollector {
+
+ public DictionaryBasedResultCollector(BlockExecutionInfo blockExecutionInfos) {
+ super(blockExecutionInfos);
+ }
+
+ /**
+ * This method will add a record both key and value to list object
+ * it will keep track of how many record is processed, to handle limit scenario
+ */
+ @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) {
+
+ List<Object[]> listBasedResult = new ArrayList<>(batchSize);
+ boolean isMsrsPresent = measureDatatypes.length > 0;
+
+ QueryDimension[] queryDimensions = tableBlockExecutionInfos.getQueryDimensions();
+ List<Integer> dictionaryIndexes = new ArrayList<Integer>();
+ for (int i = 0; i < queryDimensions.length; i++) {
+ if(queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY) ||
+ queryDimensions[i].getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY) ) {
+ dictionaryIndexes.add(queryDimensions[i].getDimension().getOrdinal());
+ }
+ }
+ int[] primitive = ArrayUtils.toPrimitive(dictionaryIndexes.toArray(
+ new Integer[dictionaryIndexes.size()]));
+ Arrays.sort(primitive);
+ int[] actualIndexInSurrogateKey = new int[dictionaryIndexes.size()];
+ int index = 0;
+ for (int i = 0; i < queryDimensions.length; i++) {
+ if(queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY) ||
+ queryDimensions[i].getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY) ) {
+ actualIndexInSurrogateKey[index++] = Arrays.binarySearch(primitive,
+ queryDimensions[i].getDimension().getOrdinal());
+ }
+ }
+
+ QueryMeasure[] queryMeasures = tableBlockExecutionInfos.getQueryMeasures();
+ BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache =
+ scannedResult.getDeleteDeltaDataCache();
+ Map<Integer, GenericQueryType> comlexDimensionInfoMap =
+ tableBlockExecutionInfos.getComlexDimensionInfoMap();
+ boolean[] dictionaryEncodingArray = CarbonUtil.getDictionaryEncodingArray(queryDimensions);
+ boolean[] directDictionaryEncodingArray =
+ CarbonUtil.getDirectDictionaryEncodingArray(queryDimensions);
+ boolean[] implictColumnArray = CarbonUtil.getImplicitColumnArray(queryDimensions);
+ boolean[] complexDataTypeArray = CarbonUtil.getComplexDataTypeArray(queryDimensions);
+ int dimSize = queryDimensions.length;
+ boolean isDimensionsExist = dimSize > 0;
+ int[] order = new int[dimSize + queryMeasures.length];
+ for (int i = 0; i < dimSize; i++) {
+ order[i] = queryDimensions[i].getQueryOrder();
+ }
+ for (int i = 0; i < queryMeasures.length; i++) {
+ order[i + dimSize] = queryMeasures[i].getQueryOrder();
+ }
+ // scan the record and add to list
+ int rowCounter = 0;
+ int dictionaryColumnIndex = 0;
+ int noDictionaryColumnIndex = 0;
+ int complexTypeColumnIndex = 0;
+ int[] surrogateResult;
+ String[] noDictionaryKeys;
+ byte[][] complexTypeKeyArray;
+ while (scannedResult.hasNext() && rowCounter < batchSize) {
+ Object[] row = new Object[dimSize + queryMeasures.length];
+ if (isDimensionsExist) {
+ surrogateResult = scannedResult.getDictionaryKeyIntegerArray();
+ noDictionaryKeys = scannedResult.getNoDictionaryKeyStringArray();
+ complexTypeKeyArray = scannedResult.getComplexTypeKeyArray();
+ dictionaryColumnIndex = 0;
+ noDictionaryColumnIndex = 0;
+ complexTypeColumnIndex = 0;
+ for (int i = 0; i < dimSize; i++) {
+ if (!dictionaryEncodingArray[i]) {
+ if (implictColumnArray[i]) {
+ if (CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID
+ .equals(queryDimensions[i].getDimension().getColName())) {
+ row[order[i]] = DataTypeUtil.getDataBasedOnDataType(
+ scannedResult.getBlockletId() + CarbonCommonConstants.FILE_SEPARATOR
+ + scannedResult.getCurrenrRowId(), DataType.STRING);
+ } else {
+ row[order[i]] = DataTypeUtil
+ .getDataBasedOnDataType(scannedResult.getBlockletId(), DataType.STRING);
+ }
+ } else {
+ row[order[i]] = DataTypeUtil
+ .getDataBasedOnDataType(noDictionaryKeys[noDictionaryColumnIndex++],
+ queryDimensions[i].getDimension().getDataType());
+ }
+ } else if (directDictionaryEncodingArray[i]) {
+ DirectDictionaryGenerator directDictionaryGenerator =
+ DirectDictionaryKeyGeneratorFactory
+ .getDirectDictionaryGenerator(queryDimensions[i].getDimension().getDataType());
+ if (directDictionaryGenerator != null) {
+ row[order[i]] = directDictionaryGenerator.getValueFromSurrogate(
+ surrogateResult[actualIndexInSurrogateKey[dictionaryColumnIndex++]]);
+ }
+ } else if (complexDataTypeArray[i]) {
+ row[order[i]] = comlexDimensionInfoMap
+ .get(queryDimensions[i].getDimension().getOrdinal())
+ .getDataBasedOnDataTypeFromSurrogates(
+ ByteBuffer.wrap(complexTypeKeyArray[complexTypeColumnIndex++]));
+ } else {
+ row[order[i]] = surrogateResult[actualIndexInSurrogateKey[dictionaryColumnIndex++]];
+ }
+ }
+
+ } else {
+ scannedResult.incrementCounter();
+ }
+ if (null != deleteDeltaDataCache && deleteDeltaDataCache
+ .contains(scannedResult.getCurrenrRowId())) {
+ continue;
+ }
+ if (isMsrsPresent) {
+ Object[] msrValues = new Object[measureDatatypes.length];
+ fillMeasureData(msrValues, 0, scannedResult);
+ for (int i = 0; i < msrValues.length; i++) {
+ row[order[i + dimSize]] = msrValues[i];
+ }
+ }
+ listBasedResult.add(row);
+ rowCounter++;
+ }
+ return listBasedResult;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java
new file mode 100644
index 0000000..e4d6d68
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.collector.impl;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.carbondata.core.metadata.Encoding;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.model.QueryDimension;
+import org.apache.carbondata.core.scan.model.QueryMeasure;
+import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
+import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
+import org.apache.carbondata.core.scan.result.vector.MeasureDataVectorProcessor;
+
+/**
+ * It is not a collector it is just a scanned result holder.
+ */
+public class DictionaryBasedVectorResultCollector extends AbstractScannedResultCollector {
+
+ private ColumnVectorInfo[] dictionaryInfo;
+
+ private ColumnVectorInfo[] noDictionaryInfo;
+
+ private ColumnVectorInfo[] complexInfo;
+
+ private ColumnVectorInfo[] measureInfo;
+
+ private ColumnVectorInfo[] allColumnInfo;
+
+ public DictionaryBasedVectorResultCollector(BlockExecutionInfo blockExecutionInfos) {
+ super(blockExecutionInfos);
+ QueryDimension[] queryDimensions = tableBlockExecutionInfos.getQueryDimensions();
+ QueryMeasure[] queryMeasures = tableBlockExecutionInfos.getQueryMeasures();
+ measureInfo = new ColumnVectorInfo[queryMeasures.length];
+ allColumnInfo = new ColumnVectorInfo[queryDimensions.length + queryMeasures.length];
+ List<ColumnVectorInfo> dictInfoList = new ArrayList<>();
+ List<ColumnVectorInfo> noDictInfoList = new ArrayList<>();
+ List<ColumnVectorInfo> complexList = new ArrayList<>();
+ for (int i = 0; i < queryDimensions.length; i++) {
+ if (!queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY)) {
+ ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo();
+ noDictInfoList.add(columnVectorInfo);
+ columnVectorInfo.dimension = queryDimensions[i];
+ columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal();
+ allColumnInfo[queryDimensions[i].getQueryOrder()] = columnVectorInfo;
+ } else if (queryDimensions[i].getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+ ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo();
+ dictInfoList.add(columnVectorInfo);
+ columnVectorInfo.dimension = queryDimensions[i];
+ columnVectorInfo.directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
+ .getDirectDictionaryGenerator(queryDimensions[i].getDimension().getDataType());
+ columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal();
+ allColumnInfo[queryDimensions[i].getQueryOrder()] = columnVectorInfo;
+ } else if (queryDimensions[i].getDimension().isComplex()) {
+ ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo();
+ complexList.add(columnVectorInfo);
+ columnVectorInfo.dimension = queryDimensions[i];
+ columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal();
+ columnVectorInfo.genericQueryType =
+ tableBlockExecutionInfos.getComlexDimensionInfoMap().get(columnVectorInfo.ordinal);
+ allColumnInfo[queryDimensions[i].getQueryOrder()] = columnVectorInfo;
+ } else {
+ ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo();
+ dictInfoList.add(columnVectorInfo);
+ columnVectorInfo.dimension = queryDimensions[i];
+ columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal();
+ allColumnInfo[queryDimensions[i].getQueryOrder()] = columnVectorInfo;
+ }
+ }
+ for (int i = 0; i < queryMeasures.length; i++) {
+ ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo();
+ columnVectorInfo.measureVectorFiller = MeasureDataVectorProcessor.MeasureVectorFillerFactory
+ .getMeasureVectorFiller(queryMeasures[i].getMeasure().getDataType());
+ columnVectorInfo.ordinal = queryMeasures[i].getMeasure().getOrdinal();
+ columnVectorInfo.measure = queryMeasures[i];
+ measureInfo[i] = columnVectorInfo;
+ allColumnInfo[queryMeasures[i].getQueryOrder()] = columnVectorInfo;
+ }
+ dictionaryInfo = dictInfoList.toArray(new ColumnVectorInfo[dictInfoList.size()]);
+ noDictionaryInfo = noDictInfoList.toArray(new ColumnVectorInfo[noDictInfoList.size()]);
+ complexInfo = complexList.toArray(new ColumnVectorInfo[complexList.size()]);
+ Arrays.sort(dictionaryInfo);
+ Arrays.sort(noDictionaryInfo);
+ Arrays.sort(complexInfo);
+ }
+
+ @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) {
+ throw new UnsupportedOperationException("collectData is not supported here");
+ }
+
+ @Override public void collectVectorBatch(AbstractScannedResult scannedResult,
+ CarbonColumnarBatch columnarBatch) {
+ int rowCounter = scannedResult.getRowCounter();
+ int availableRows = scannedResult.numberOfOutputRows() - rowCounter;
+ int requiredRows = columnarBatch.getBatchSize() - columnarBatch.getActualSize();
+ requiredRows = Math.min(requiredRows, availableRows);
+ if (requiredRows < 1) {
+ return;
+ }
+ for (int i = 0; i < allColumnInfo.length; i++) {
+ allColumnInfo[i].size = requiredRows;
+ allColumnInfo[i].offset = rowCounter;
+ allColumnInfo[i].vectorOffset = columnarBatch.getRowCounter();
+ allColumnInfo[i].vector = columnarBatch.columnVectors[i];
+ }
+
+ scannedResult.fillColumnarDictionaryBatch(dictionaryInfo);
+ scannedResult.fillColumnarNoDictionaryBatch(noDictionaryInfo);
+ scannedResult.fillColumnarMeasureBatch(measureInfo, measuresOrdinal);
+ scannedResult.fillColumnarComplexBatch(complexInfo);
+ scannedResult.setRowCounter(rowCounter + requiredRows);
+ columnarBatch.setActualSize(columnarBatch.getActualSize() + requiredRows);
+ columnarBatch.setRowCounter(columnarBatch.getRowCounter() + requiredRows);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java
new file mode 100644
index 0000000..cf42d9c
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.collector.impl;
+
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.core.update.data.BlockletLevelDeleteDeltaDataCache;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.model.QueryMeasure;
+import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
+
+
+/**
+ * It is not a collector it is just a scanned result holder.
+ */
+public class RawBasedResultCollector extends AbstractScannedResultCollector {
+
+ public RawBasedResultCollector(BlockExecutionInfo blockExecutionInfos) {
+ super(blockExecutionInfos);
+ }
+
+ /**
+ * This method will add a record both key and value to list object
+ * it will keep track of how many record is processed, to handle limit scenario
+ */
+ @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) {
+ List<Object[]> listBasedResult = new ArrayList<>(batchSize);
+ QueryMeasure[] queryMeasures = tableBlockExecutionInfos.getQueryMeasures();
+ ByteArrayWrapper wrapper = null;
+ BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache =
+ scannedResult.getDeleteDeltaDataCache();
+ // scan the record and add to list
+ int rowCounter = 0;
+ while (scannedResult.hasNext() && rowCounter < batchSize) {
+ Object[] row = new Object[1 + queryMeasures.length];
+ wrapper = new ByteArrayWrapper();
+ wrapper.setDictionaryKey(scannedResult.getDictionaryKeyArray());
+ wrapper.setNoDictionaryKeys(scannedResult.getNoDictionaryKeyArray());
+ wrapper.setComplexTypesKeys(scannedResult.getComplexTypeKeyArray());
+ wrapper.setImplicitColumnByteArray(scannedResult.getBlockletId()
+ .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
+ if (null != deleteDeltaDataCache && deleteDeltaDataCache
+ .contains(scannedResult.getCurrenrRowId())) {
+ continue;
+ }
+ row[0] = wrapper;
+ fillMeasureData(row, 1, scannedResult);
+ listBasedResult.add(row);
+ rowCounter++;
+ }
+ updateData(listBasedResult);
+ return listBasedResult;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java
new file mode 100644
index 0000000..39dac3a
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ArrayQueryType.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.complextypes;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.scan.filter.GenericQueryType;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+
+public class ArrayQueryType extends ComplexQueryType implements GenericQueryType {
+
+ private GenericQueryType children;
+
+ public ArrayQueryType(String name, String parentname, int blockIndex) {
+ super(name, parentname, blockIndex);
+ }
+
+ @Override public void addChildren(GenericQueryType children) {
+ if (this.getName().equals(children.getParentname())) {
+ this.children = children;
+ } else {
+ this.children.addChildren(children);
+ }
+ }
+
+ @Override public String getName() {
+ return name;
+ }
+
+ @Override public void setName(String name) {
+ this.name = name;
+ }
+
+ @Override public String getParentname() {
+ return parentname;
+ }
+
+ @Override public void setParentname(String parentname) {
+ this.parentname = parentname;
+
+ }
+
+ public void parseBlocksAndReturnComplexColumnByteArray(
+ DimensionColumnDataChunk[] dimensionColumnDataChunks, int rowNumber,
+ DataOutputStream dataOutputStream) throws IOException {
+ byte[] input = new byte[8];
+ copyBlockDataChunk(dimensionColumnDataChunks, rowNumber, input);
+ ByteBuffer byteArray = ByteBuffer.wrap(input);
+ int dataLength = byteArray.getInt();
+ dataOutputStream.writeInt(dataLength);
+ if (dataLength > 0) {
+ int columnIndex = byteArray.getInt();
+ for (int i = 0; i < dataLength; i++) {
+ children
+ .parseBlocksAndReturnComplexColumnByteArray(dimensionColumnDataChunks, columnIndex++,
+ dataOutputStream);
+ }
+ }
+ }
+
+ @Override public int getColsCount() {
+ return children.getColsCount() + 1;
+ }
+
+ @Override public void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder)
+ throws IOException {
+ readBlockDataChunk(blockChunkHolder);
+ children.fillRequiredBlockData(blockChunkHolder);
+ }
+
+ @Override public Object getDataBasedOnDataTypeFromSurrogates(ByteBuffer surrogateData) {
+ int dataLength = surrogateData.getInt();
+ if (dataLength == -1) {
+ return null;
+ }
+ Object[] data = new Object[dataLength];
+ for (int i = 0; i < dataLength; i++) {
+ data[i] = children.getDataBasedOnDataTypeFromSurrogates(surrogateData);
+ }
+ return data;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java
new file mode 100644
index 0000000..75ed3ff
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/ComplexQueryType.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.complextypes;
+
+import java.io.IOException;
+
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.scan.filter.GenericQueryType;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+
+public class ComplexQueryType {
+ protected GenericQueryType children;
+
+ protected String name;
+
+ protected String parentname;
+
+ protected int blockIndex;
+
+ public ComplexQueryType(String name, String parentname, int blockIndex) {
+ this.name = name;
+ this.parentname = parentname;
+ this.blockIndex = blockIndex;
+ }
+
+ /**
+ * Method will copy the block chunk holder data to the passed
+ * byte[], this method is also used by child
+ *
+ * @param rowNumber
+ * @param input
+ */
+ protected void copyBlockDataChunk(DimensionColumnDataChunk[] dimensionColumnDataChunks,
+ int rowNumber, byte[] input) {
+ byte[] data = dimensionColumnDataChunks[blockIndex].getChunkData(rowNumber);
+ System.arraycopy(data, 0, input, 0, data.length);
+ }
+
+ /*
+ * This method will read the block data chunk from the respective block
+ */
+ protected void readBlockDataChunk(BlocksChunkHolder blockChunkHolder) throws IOException {
+ if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
+ blockChunkHolder.getDimensionDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+ .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java
new file mode 100644
index 0000000..9ef5fad
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.complextypes;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.core.cache.dictionary.Dictionary;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
+import org.apache.carbondata.core.keygenerator.mdkey.Bits;
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.core.scan.filter.GenericQueryType;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+
+public class PrimitiveQueryType extends ComplexQueryType implements GenericQueryType {
+
+ private String name;
+ private String parentname;
+
+ private int keySize;
+
+ private int blockIndex;
+
+ private Dictionary dictionary;
+
+ private DataType dataType;
+
+ private boolean isDirectDictionary;
+
+ public PrimitiveQueryType(String name, String parentname, int blockIndex, DataType dataType,
+ int keySize, Dictionary dictionary, boolean isDirectDictionary) {
+ super(name, parentname, blockIndex);
+ this.dataType = dataType;
+ this.keySize = keySize;
+ this.dictionary = dictionary;
+ this.name = name;
+ this.parentname = parentname;
+ this.blockIndex = blockIndex;
+ this.isDirectDictionary = isDirectDictionary;
+ }
+
+ @Override public void addChildren(GenericQueryType children) {
+
+ }
+
+ @Override public String getName() {
+ return name;
+ }
+
+ @Override public void setName(String name) {
+ this.name = name;
+ }
+
+ @Override public String getParentname() {
+ return parentname;
+ }
+
+ @Override public void setParentname(String parentname) {
+ this.parentname = parentname;
+
+ }
+
+ @Override public int getColsCount() {
+ return 1;
+ }
+
+ @Override public void parseBlocksAndReturnComplexColumnByteArray(
+ DimensionColumnDataChunk[] dimensionDataChunks, int rowNumber,
+ DataOutputStream dataOutputStream) throws IOException {
+ byte[] currentVal =
+ new byte[dimensionDataChunks[blockIndex].getColumnValueSize()];
+ copyBlockDataChunk(dimensionDataChunks, rowNumber, currentVal);
+ dataOutputStream.write(currentVal);
+ }
+
+ @Override public void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder)
+ throws IOException {
+ readBlockDataChunk(blockChunkHolder);
+ }
+
+ @Override public Object getDataBasedOnDataTypeFromSurrogates(ByteBuffer surrogateData) {
+ byte[] data = new byte[keySize];
+ surrogateData.get(data);
+ Bits bit = new Bits(new int[]{keySize * 8});
+ int surrgateValue = (int)bit.getKeyArray(data, 0)[0];
+ Object actualData = null;
+ if (isDirectDictionary) {
+ DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
+ .getDirectDictionaryGenerator(dataType);
+ actualData = directDictionaryGenerator.getValueFromSurrogate(surrgateValue);
+ } else {
+ String dictionaryValueForKey = dictionary.getDictionaryValueForKey(surrgateValue);
+ actualData = DataTypeUtil.getDataBasedOnDataType(dictionaryValueForKey, this.dataType);
+ }
+ return actualData;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java
new file mode 100644
index 0000000..bbb1cac
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/StructQueryType.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.complextypes;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.scan.filter.GenericQueryType;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+
+public class StructQueryType extends ComplexQueryType implements GenericQueryType {
+
+ private List<GenericQueryType> children = new ArrayList<GenericQueryType>();
+ private String name;
+ private String parentname;
+
+ public StructQueryType(String name, String parentname, int blockIndex) {
+ super(name, parentname, blockIndex);
+ this.name = name;
+ this.parentname = parentname;
+ }
+
+ @Override public void addChildren(GenericQueryType newChild) {
+ if (this.getName().equals(newChild.getParentname())) {
+ this.children.add(newChild);
+ } else {
+ for (GenericQueryType child : this.children) {
+ child.addChildren(newChild);
+ }
+ }
+
+ }
+
+ @Override public String getName() {
+ return name;
+ }
+
+ @Override public void setName(String name) {
+ this.name = name;
+ }
+
+ @Override public String getParentname() {
+ return parentname;
+ }
+
+ @Override public void setParentname(String parentname) {
+ this.parentname = parentname;
+
+ }
+
+ @Override public int getColsCount() {
+ int colsCount = 1;
+ for (int i = 0; i < children.size(); i++) {
+ colsCount += children.get(i).getColsCount();
+ }
+ return colsCount;
+ }
+
+ @Override public void parseBlocksAndReturnComplexColumnByteArray(
+ DimensionColumnDataChunk[] dimensionColumnDataChunks, int rowNumber,
+ DataOutputStream dataOutputStream) throws IOException {
+ byte[] input = new byte[8];
+ copyBlockDataChunk(dimensionColumnDataChunks, rowNumber, input);
+ ByteBuffer byteArray = ByteBuffer.wrap(input);
+ int childElement = byteArray.getInt();
+ dataOutputStream.writeInt(childElement);
+ if (childElement > 0){
+ for (int i = 0; i < childElement; i++) {
+ children.get(i)
+ .parseBlocksAndReturnComplexColumnByteArray(dimensionColumnDataChunks, rowNumber,
+ dataOutputStream);
+ }
+ }
+ }
+
+ @Override public void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder)
+ throws IOException {
+ readBlockDataChunk(blockChunkHolder);
+
+ for (int i = 0; i < children.size(); i++) {
+ children.get(i).fillRequiredBlockData(blockChunkHolder);
+ }
+ }
+
+ @Override public Object getDataBasedOnDataTypeFromSurrogates(ByteBuffer surrogateData) {
+ int childLength = surrogateData.getInt();
+ Object[] fields = new Object[childLength];
+ for (int i = 0; i < childLength; i++) {
+ fields[i] = children.get(i).getDataBasedOnDataTypeFromSurrogates(surrogateData);
+ }
+ return fields;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/executor/QueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/QueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/QueryExecutor.java
new file mode 100644
index 0000000..7b324a2
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/QueryExecutor.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.executor;
+
+import java.io.IOException;
+
+import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
+import org.apache.carbondata.core.scan.model.QueryModel;
+
+/**
+ * Interface for carbon query executor.
+ * Will be used to execute the query based on the query model
+ * and will return the iterator over query result
+ */
+public interface QueryExecutor<E> {
+
+ /**
+ * Below method will be used to execute the query based on query model passed from driver
+ *
+ * @param queryModel query details
+ * @return query result iterator
+ * @throws QueryExecutionException if any failure while executing the query
+ * @throws IOException if fail to read files
+ */
+ CarbonIterator<E> execute(QueryModel queryModel)
+ throws QueryExecutionException, IOException;
+
+ /**
+ * Below method will be used for cleanup
+ *
+ * @throws QueryExecutionException
+ */
+ void finish() throws QueryExecutionException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/executor/QueryExecutorFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/QueryExecutorFactory.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/QueryExecutorFactory.java
new file mode 100644
index 0000000..293129b
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/QueryExecutorFactory.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.executor;
+
+import org.apache.carbondata.core.scan.executor.impl.DetailQueryExecutor;
+import org.apache.carbondata.core.scan.executor.impl.VectorDetailQueryExecutor;
+import org.apache.carbondata.core.scan.model.QueryModel;
+
+/**
+ * Factory class to get the query executor from RDD
+ * This will return the executor based on query type
+ */
+public class QueryExecutorFactory {
+
+ public static QueryExecutor getQueryExecutor(QueryModel queryModel) {
+ if (queryModel.isVectorReader()) {
+ return new VectorDetailQueryExecutor();
+ } else {
+ return new DetailQueryExecutor();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/executor/exception/QueryExecutionException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/exception/QueryExecutionException.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/exception/QueryExecutionException.java
new file mode 100644
index 0000000..0ac1f05
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/exception/QueryExecutionException.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.executor.exception;
+
+import java.util.Locale;
+
+/**
+ * Exception class for query execution
+ *
+ * @author Administrator
+ */
+public class QueryExecutionException extends Exception {
+
+ /**
+ * default serial version ID.
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * The Error message.
+ */
+ private String msg = "";
+
+ /**
+ * Constructor
+ *
+ * @param msg The error message for this exception.
+ */
+ public QueryExecutionException(String msg) {
+ super(msg);
+ this.msg = msg;
+ }
+
+ /**
+ * Constructor
+ *
+ * @param msg The error message for this exception.
+ */
+ public QueryExecutionException(String msg, Throwable t) {
+ super(msg, t);
+ this.msg = msg;
+ }
+
+ /**
+ * Constructor
+ *
+ * @param t
+ */
+ public QueryExecutionException(Throwable t) {
+ super(t);
+ }
+
+ /**
+ * This method is used to get the localized message.
+ *
+ * @param locale - A Locale object represents a specific geographical,
+ * political, or cultural region.
+ * @return - Localized error message.
+ */
+ public String getLocalizedMessage(Locale locale) {
+ return "";
+ }
+
+ /**
+ * getLocalizedMessage
+ */
+ @Override public String getLocalizedMessage() {
+ return super.getLocalizedMessage();
+ }
+
+ /**
+ * getMessage
+ */
+ public String getMessage() {
+ return this.msg;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
new file mode 100644
index 0000000..05df636
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -0,0 +1,483 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.executor.impl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.common.logging.impl.StandardLogService;
+import org.apache.carbondata.core.cache.CacheProvider;
+import org.apache.carbondata.core.cache.CacheType;
+import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.datastore.BlockIndexStore;
+import org.apache.carbondata.core.datastore.IndexKey;
+import org.apache.carbondata.core.datastore.block.AbstractIndex;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.datastore.block.TableBlockUniqueIdentifier;
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.metadata.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
+import org.apache.carbondata.core.path.CarbonStorePath;
+import org.apache.carbondata.core.stats.QueryStatistic;
+import org.apache.carbondata.core.stats.QueryStatisticsConstants;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.keygenerator.KeyGenerator;
+import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.scan.executor.QueryExecutor;
+import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
+import org.apache.carbondata.core.scan.executor.infos.AggregatorInfo;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.executor.util.QueryUtil;
+import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.model.QueryDimension;
+import org.apache.carbondata.core.scan.model.QueryMeasure;
+import org.apache.carbondata.core.scan.model.QueryModel;
+
+import org.apache.commons.lang3.ArrayUtils;
+
+/**
+ * This class provides a skeletal implementation of the {@link QueryExecutor}
+ * interface to minimize the effort required to implement this interface. This
+ * will be used to prepare all the properties required for query execution
+ */
+public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(AbstractQueryExecutor.class.getName());
+ /**
+ * holder for query properties which will be used to execute the query
+ */
+ protected QueryExecutorProperties queryProperties;
+
+ public AbstractQueryExecutor() {
+ queryProperties = new QueryExecutorProperties();
+ }
+
+ /**
+ * Below method will be used to fill the executor properties based on query
+ * model it will parse the query model and get the detail and fill it in
+ * query properties
+ *
+ * @param queryModel
+ */
+ protected void initQuery(QueryModel queryModel) throws IOException {
+ StandardLogService.setThreadName(StandardLogService.getPartitionID(
+ queryModel.getAbsoluteTableIdentifier().getCarbonTableIdentifier().getTableName()),
+ queryModel.getQueryId());
+ LOGGER.info("Query will be executed on table: " + queryModel.getAbsoluteTableIdentifier()
+ .getCarbonTableIdentifier().getTableName());
+ // add executor service for query execution
+ queryProperties.executorService = Executors.newFixedThreadPool(1);
+ // Initializing statistics list to record the query statistics
+ // creating copy on write to handle concurrent scenario
+ queryProperties.queryStatisticsRecorder =
+ CarbonTimeStatisticsFactory.createExecutorRecorder(queryModel.getQueryId());
+ queryModel.setStatisticsRecorder(queryProperties.queryStatisticsRecorder);
+ QueryUtil.resolveQueryModel(queryModel);
+ QueryStatistic queryStatistic = new QueryStatistic();
+ // sort the block info
+ // so block will be loaded in sorted order this will be required for
+ // query execution
+ Collections.sort(queryModel.getTableBlockInfos());
+ // get the table blocks
+ CacheProvider cacheProvider = CacheProvider.getInstance();
+ BlockIndexStore<TableBlockUniqueIdentifier, AbstractIndex> cache =
+ (BlockIndexStore) cacheProvider
+ .createCache(CacheType.EXECUTOR_BTREE, queryModel.getTable().getStorePath());
+ // remove the invalid table blocks, block which is deleted or compacted
+ cache.removeTableBlocks(queryModel.getInvalidSegmentIds(),
+ queryModel.getAbsoluteTableIdentifier());
+ List<TableBlockUniqueIdentifier> tableBlockUniqueIdentifiers =
+ prepareTableBlockUniqueIdentifier(queryModel.getTableBlockInfos(),
+ queryModel.getAbsoluteTableIdentifier());
+ cache.removeTableBlocksIfHorizontalCompactionDone(queryModel);
+ queryProperties.dataBlocks = cache.getAll(tableBlockUniqueIdentifiers);
+ queryStatistic
+ .addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_EXECUTOR, System.currentTimeMillis());
+ queryProperties.queryStatisticsRecorder.recordStatistics(queryStatistic);
+ //
+ // // updating the restructuring infos for the query
+ queryProperties.keyStructureInfo = getKeyStructureInfo(queryModel,
+ queryProperties.dataBlocks.get(queryProperties.dataBlocks.size() - 1).getSegmentProperties()
+ .getDimensionKeyGenerator());
+
+ // calculating the total number of aggeragted columns
+ int aggTypeCount = queryModel.getQueryMeasures().size();
+
+ int currentIndex = 0;
+ DataType[] dataTypes = new DataType[aggTypeCount];
+
+ for (QueryMeasure carbonMeasure : queryModel.getQueryMeasures()) {
+ // adding the data type and aggregation type of all the measure this
+ // can be used
+ // to select the aggregator
+ dataTypes[currentIndex] = carbonMeasure.getMeasure().getDataType();
+ currentIndex++;
+ }
+ queryProperties.measureDataTypes = dataTypes;
+ // as aggregation will be executed in following order
+ // 1.aggregate dimension expression
+ // 2. expression
+ // 3. query measure
+ // so calculating the index of the expression start index
+ // and measure column start index
+ queryProperties.filterMeasures = new HashSet<>();
+ queryProperties.complexFilterDimension = new HashSet<>();
+ QueryUtil.getAllFilterDimensions(queryModel.getFilterExpressionResolverTree(),
+ queryProperties.complexFilterDimension, queryProperties.filterMeasures);
+
+ queryStatistic = new QueryStatistic();
+ // dictionary column unique column id to dictionary mapping
+ // which will be used to get column actual data
+ queryProperties.columnToDictionayMapping = QueryUtil
+ .getDimensionDictionaryDetail(queryModel.getQueryDimension(),
+ queryProperties.complexFilterDimension, queryModel.getAbsoluteTableIdentifier());
+ queryStatistic
+ .addStatistics(QueryStatisticsConstants.LOAD_DICTIONARY, System.currentTimeMillis());
+ queryProperties.queryStatisticsRecorder.recordStatistics(queryStatistic);
+ queryModel.setColumnToDictionaryMapping(queryProperties.columnToDictionayMapping);
+ }
+
+ private List<TableBlockUniqueIdentifier> prepareTableBlockUniqueIdentifier(
+ List<TableBlockInfo> tableBlockInfos, AbsoluteTableIdentifier absoluteTableIdentifier) {
+ List<TableBlockUniqueIdentifier> tableBlockUniqueIdentifiers =
+ new ArrayList<>(tableBlockInfos.size());
+ for (TableBlockInfo blockInfo : tableBlockInfos) {
+ tableBlockUniqueIdentifiers
+ .add(new TableBlockUniqueIdentifier(absoluteTableIdentifier, blockInfo));
+ }
+ return tableBlockUniqueIdentifiers;
+ }
+
+ /**
+ * Below method will be used to get the key structure info for the query
+ *
+ * @param queryModel query model
+ * @param keyGenerator
+ * @return key structure info
+ */
+ private KeyStructureInfo getKeyStructureInfo(QueryModel queryModel, KeyGenerator keyGenerator) {
+ // getting the masked byte range for dictionary column
+ int[] maskByteRanges =
+ QueryUtil.getMaskedByteRange(queryModel.getQueryDimension(), keyGenerator);
+
+ // max key for the dictionary dimension present in the query
+ byte[] maxKey = null;
+ try {
+ // getting the max key which will be used to masked and get the
+ // masked key
+ maxKey = QueryUtil.getMaxKeyBasedOnDimensions(queryModel.getQueryDimension(), keyGenerator);
+ } catch (KeyGenException e) {
+ LOGGER.error(e, "problem while getting the max key");
+ }
+
+ KeyStructureInfo restructureInfos = new KeyStructureInfo();
+ restructureInfos.setKeyGenerator(keyGenerator);
+ restructureInfos.setMaskByteRanges(maskByteRanges);
+ restructureInfos.setMaxKey(maxKey);
+ return restructureInfos;
+ }
+
+ protected List<BlockExecutionInfo> getBlockExecutionInfos(QueryModel queryModel)
+ throws IOException, QueryExecutionException {
+ initQuery(queryModel);
+ List<BlockExecutionInfo> blockExecutionInfoList = new ArrayList<BlockExecutionInfo>();
+ // fill all the block execution infos for all the blocks selected in
+ // query
+ // and query will be executed based on that infos
+ for (int i = 0; i < queryProperties.dataBlocks.size(); i++) {
+ blockExecutionInfoList.add(
+ getBlockExecutionInfoForBlock(queryModel, queryProperties.dataBlocks.get(i),
+ queryModel.getTableBlockInfos().get(i).getBlockletInfos().getStartBlockletNumber(),
+ queryModel.getTableBlockInfos().get(i).getBlockletInfos().getNumberOfBlockletToScan(),
+ queryModel.getTableBlockInfos().get(i).getFilePath()));
+ }
+ if (null != queryModel.getStatisticsRecorder()) {
+ QueryStatistic queryStatistic = new QueryStatistic();
+ queryStatistic.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKS_NUM,
+ blockExecutionInfoList.size());
+ queryModel.getStatisticsRecorder().recordStatistics(queryStatistic);
+ }
+ return blockExecutionInfoList;
+ }
+
+ /**
+ * Below method will be used to get the block execution info which is
+ * required to execute any block based on query model
+ *
+ * @param queryModel query model from user query
+ * @param blockIndex block index
+ * @return block execution info
+ * @throws QueryExecutionException any failure during block info creation
+ */
+ protected BlockExecutionInfo getBlockExecutionInfoForBlock(QueryModel queryModel,
+ AbstractIndex blockIndex, int startBlockletIndex, int numberOfBlockletToScan, String filePath)
+ throws QueryExecutionException {
+ BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
+ SegmentProperties segmentProperties = blockIndex.getSegmentProperties();
+ List<CarbonDimension> tableBlockDimensions = segmentProperties.getDimensions();
+ KeyGenerator blockKeyGenerator = segmentProperties.getDimensionKeyGenerator();
+
+ // below is to get only those dimension in query which is present in the
+ // table block
+ List<QueryDimension> updatedQueryDimension = RestructureUtil
+ .getUpdatedQueryDimension(queryModel.getQueryDimension(), tableBlockDimensions,
+ segmentProperties.getComplexDimensions());
+ // TODO add complex dimension children
+ int[] maskByteRangesForBlock =
+ QueryUtil.getMaskedByteRange(updatedQueryDimension, blockKeyGenerator);
+ int[] maksedByte =
+ QueryUtil.getMaskedByte(blockKeyGenerator.getKeySizeInBytes(), maskByteRangesForBlock);
+ int tableFactPathLength = CarbonStorePath
+ .getCarbonTablePath(queryModel.getAbsoluteTableIdentifier().getStorePath(),
+ queryModel.getAbsoluteTableIdentifier().getCarbonTableIdentifier()).getFactDir()
+ .length() + 1;
+ blockExecutionInfo.setBlockId(filePath.substring(tableFactPathLength));
+ blockExecutionInfo.setStartBlockletIndex(startBlockletIndex);
+ blockExecutionInfo.setNumberOfBlockletToScan(numberOfBlockletToScan);
+ blockExecutionInfo.setQueryDimensions(
+ updatedQueryDimension.toArray(new QueryDimension[updatedQueryDimension.size()]));
+ blockExecutionInfo.setQueryMeasures(queryModel.getQueryMeasures()
+ .toArray(new QueryMeasure[queryModel.getQueryMeasures().size()]));
+ blockExecutionInfo.setDataBlock(blockIndex);
+ blockExecutionInfo.setBlockKeyGenerator(blockKeyGenerator);
+ // adding aggregation info for query
+ blockExecutionInfo.setAggregatorInfo(getAggregatorInfoForBlock(queryModel, blockIndex));
+ // setting whether raw record query or not
+ blockExecutionInfo.setRawRecordDetailQuery(queryModel.isForcedDetailRawQuery());
+ // setting the masked byte of the block which will be
+ // used to update the unpack the older block keys
+ blockExecutionInfo.setMaskedByteForBlock(maksedByte);
+ // total number dimension
+ blockExecutionInfo
+ .setTotalNumberDimensionBlock(segmentProperties.getDimensionOrdinalToBlockMapping().size());
+ blockExecutionInfo
+ .setTotalNumberOfMeasureBlock(segmentProperties.getMeasuresOrdinalToBlockMapping().size());
+ blockExecutionInfo.setAbsoluteTableIdentifier(queryModel.getAbsoluteTableIdentifier());
+ blockExecutionInfo.setComplexDimensionInfoMap(QueryUtil
+ .getComplexDimensionsMap(updatedQueryDimension,
+ segmentProperties.getDimensionOrdinalToBlockMapping(),
+ segmentProperties.getEachComplexDimColumnValueSize(),
+ queryProperties.columnToDictionayMapping, queryProperties.complexFilterDimension));
+ // to check whether older block key update is required or not
+ blockExecutionInfo.setFixedKeyUpdateRequired(
+ !blockKeyGenerator.equals(queryProperties.keyStructureInfo.getKeyGenerator()));
+ IndexKey startIndexKey = null;
+ IndexKey endIndexKey = null;
+ if (null != queryModel.getFilterExpressionResolverTree()) {
+ // loading the filter executer tree for filter evaluation
+ blockExecutionInfo.setFilterExecuterTree(FilterUtil
+ .getFilterExecuterTree(queryModel.getFilterExpressionResolverTree(), segmentProperties,
+ blockExecutionInfo.getComlexDimensionInfoMap()));
+ List<IndexKey> listOfStartEndKeys = new ArrayList<IndexKey>(2);
+ FilterUtil.traverseResolverTreeAndGetStartAndEndKey(segmentProperties,
+ queryModel.getFilterExpressionResolverTree(), listOfStartEndKeys);
+ startIndexKey = listOfStartEndKeys.get(0);
+ endIndexKey = listOfStartEndKeys.get(1);
+ } else {
+ try {
+ startIndexKey = FilterUtil.prepareDefaultStartIndexKey(segmentProperties);
+ endIndexKey = FilterUtil.prepareDefaultEndIndexKey(segmentProperties);
+ } catch (KeyGenException e) {
+ throw new QueryExecutionException(e);
+ }
+ }
+ //setting the start index key of the block node
+ blockExecutionInfo.setStartKey(startIndexKey);
+ //setting the end index key of the block node
+ blockExecutionInfo.setEndKey(endIndexKey);
+ // expression dimensions
+ List<CarbonDimension> expressionDimensions =
+ new ArrayList<CarbonDimension>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ // expression measure
+ List<CarbonMeasure> expressionMeasures =
+ new ArrayList<CarbonMeasure>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ // setting all the dimension chunk indexes to be read from file
+ int numberOfElementToConsider = 0;
+ // list of dimensions to be projected
+ List<Integer> allProjectionListDimensionIdexes = new ArrayList<>();
+ int[] dimensionsBlockIndexes = QueryUtil.getDimensionsBlockIndexes(updatedQueryDimension,
+ segmentProperties.getDimensionOrdinalToBlockMapping(), expressionDimensions,
+ queryProperties.complexFilterDimension, allProjectionListDimensionIdexes);
+ if (dimensionsBlockIndexes.length > 0) {
+ numberOfElementToConsider = dimensionsBlockIndexes[dimensionsBlockIndexes.length - 1]
+ == segmentProperties.getBlockTodimensionOrdinalMapping().size() - 1 ?
+ dimensionsBlockIndexes.length - 1 :
+ dimensionsBlockIndexes.length;
+ blockExecutionInfo.setAllSelectedDimensionBlocksIndexes(CarbonUtil
+ .getRangeIndex(dimensionsBlockIndexes, numberOfElementToConsider,
+ CarbonCommonConstants.NUMBER_OF_COLUMN_READ_IN_IO));
+ } else {
+ blockExecutionInfo.setAllSelectedDimensionBlocksIndexes(new int[0][0]);
+ }
+
+ int[] measureBlockIndexes = QueryUtil
+ .getMeasureBlockIndexes(queryModel.getQueryMeasures(), expressionMeasures,
+ segmentProperties.getMeasuresOrdinalToBlockMapping(), queryProperties.filterMeasures);
+ if (measureBlockIndexes.length > 0) {
+
+ numberOfElementToConsider = measureBlockIndexes[measureBlockIndexes.length - 1]
+ == segmentProperties.getMeasures().size() - 1 ?
+ measureBlockIndexes.length - 1 :
+ measureBlockIndexes.length;
+ // setting all the measure chunk indexes to be read from file
+ blockExecutionInfo.setAllSelectedMeasureBlocksIndexes(CarbonUtil
+ .getRangeIndex(measureBlockIndexes, numberOfElementToConsider,
+ CarbonCommonConstants.NUMBER_OF_COLUMN_READ_IN_IO));
+ } else {
+ blockExecutionInfo.setAllSelectedMeasureBlocksIndexes(new int[0][0]);
+ }
+ // setting the key structure info which will be required
+ // to update the older block key with new key generator
+ blockExecutionInfo.setKeyStructureInfo(queryProperties.keyStructureInfo);
+ // setting the size of fixed key column (dictionary column)
+ blockExecutionInfo.setFixedLengthKeySize(getKeySize(updatedQueryDimension, segmentProperties));
+ Set<Integer> dictionaryColumnBlockIndex = new HashSet<Integer>();
+ List<Integer> noDictionaryColumnBlockIndex = new ArrayList<Integer>();
+ // get the block index to be read from file for query dimension
+ // for both dictionary columns and no dictionary columns
+ QueryUtil.fillQueryDimensionsBlockIndexes(updatedQueryDimension,
+ segmentProperties.getDimensionOrdinalToBlockMapping(), dictionaryColumnBlockIndex,
+ noDictionaryColumnBlockIndex);
+ int[] queryDictionaryColumnBlockIndexes = ArrayUtils.toPrimitive(
+ dictionaryColumnBlockIndex.toArray(new Integer[dictionaryColumnBlockIndex.size()]));
+ // need to sort the dictionary column as for all dimension
+ // column key will be filled based on key order
+ Arrays.sort(queryDictionaryColumnBlockIndexes);
+ blockExecutionInfo.setDictionaryColumnBlockIndex(queryDictionaryColumnBlockIndexes);
+ // setting the no dictionary column block indexes
+ blockExecutionInfo.setNoDictionaryBlockIndexes(ArrayUtils.toPrimitive(
+ noDictionaryColumnBlockIndex.toArray(new Integer[noDictionaryColumnBlockIndex.size()])));
+ // setting column id to dictionary mapping
+ blockExecutionInfo.setColumnIdToDcitionaryMapping(queryProperties.columnToDictionayMapping);
+ // setting each column value size
+ blockExecutionInfo.setEachColumnValueSize(segmentProperties.getEachDimColumnValueSize());
+ blockExecutionInfo.setComplexColumnParentBlockIndexes(
+ getComplexDimensionParentBlockIndexes(updatedQueryDimension));
+ blockExecutionInfo.setVectorBatchCollector(queryModel.isVectorReader());
+ try {
+ // to set column group and its key structure info which will be used
+ // to
+ // for getting the column group column data in case of final row
+ // and in case of dimension aggregation
+ blockExecutionInfo.setColumnGroupToKeyStructureInfo(
+ QueryUtil.getColumnGroupKeyStructureInfo(updatedQueryDimension, segmentProperties));
+ } catch (KeyGenException e) {
+ throw new QueryExecutionException(e);
+ }
+ return blockExecutionInfo;
+ }
+
+ /**
+ * This method will be used to get fixed key length size this will be used
+ * to create a row from column chunk
+ *
+ * @param queryDimension query dimension
+ * @param blockMetadataInfo block metadata info
+ * @return key size
+ */
+ private int getKeySize(List<QueryDimension> queryDimension, SegmentProperties blockMetadataInfo) {
+ List<Integer> fixedLengthDimensionOrdinal =
+ new ArrayList<Integer>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ int counter = 0;
+ while (counter < queryDimension.size()) {
+ if (queryDimension.get(counter).getDimension().numberOfChild() > 0) {
+ counter += queryDimension.get(counter).getDimension().numberOfChild();
+ continue;
+ } else if (!CarbonUtil.hasEncoding(queryDimension.get(counter).getDimension().getEncoder(),
+ Encoding.DICTIONARY)) {
+ counter++;
+ } else {
+ fixedLengthDimensionOrdinal.add(queryDimension.get(counter).getDimension().getKeyOrdinal());
+ counter++;
+ }
+ }
+ int[] dictioanryColumnOrdinal = ArrayUtils.toPrimitive(
+ fixedLengthDimensionOrdinal.toArray(new Integer[fixedLengthDimensionOrdinal.size()]));
+ if (dictioanryColumnOrdinal.length > 0) {
+ return blockMetadataInfo.getFixedLengthKeySplitter()
+ .getKeySizeByBlock(dictioanryColumnOrdinal);
+ }
+ return 0;
+ }
+
+ /**
+ * Below method will be used to get the aggrgator info for the query
+ *
+ * @param queryModel query model
+ * @param tableBlock table block
+ * @return aggregator info
+ */
+ private AggregatorInfo getAggregatorInfoForBlock(QueryModel queryModel,
+ AbstractIndex tableBlock) {
+ // getting the aggregate infos which will be used during aggregation
+ AggregatorInfo aggregatorInfos = RestructureUtil
+ .getAggregatorInfos(queryModel.getQueryMeasures(),
+ tableBlock.getSegmentProperties().getMeasures());
+ // setting the measure aggregator for all aggregation function selected
+ // in query
+ aggregatorInfos.setMeasureDataTypes(queryProperties.measureDataTypes);
+ return aggregatorInfos;
+ }
+
+ private int[] getComplexDimensionParentBlockIndexes(List<QueryDimension> queryDimensions) {
+ List<Integer> parentBlockIndexList = new ArrayList<Integer>();
+ for (QueryDimension queryDimension : queryDimensions) {
+ if (CarbonUtil.hasDataType(queryDimension.getDimension().getDataType(),
+ new DataType[] { DataType.ARRAY, DataType.STRUCT, DataType.MAP })) {
+ parentBlockIndexList.add(queryDimension.getDimension().getOrdinal());
+ }
+ }
+ return ArrayUtils
+ .toPrimitive(parentBlockIndexList.toArray(new Integer[parentBlockIndexList.size()]));
+ }
+
+ /**
+ * Below method will be used to finish the execution
+ *
+ * @throws QueryExecutionException
+ */
+ @Override public void finish() throws QueryExecutionException {
+ CarbonUtil.clearBlockCache(queryProperties.dataBlocks);
+ if (null != queryProperties.executorService) {
+ queryProperties.executorService.shutdown();
+ try {
+ queryProperties.executorService.awaitTermination(1, TimeUnit.HOURS);
+ } catch (InterruptedException e) {
+ throw new QueryExecutionException(e);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/DetailQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/DetailQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/DetailQueryExecutor.java
new file mode 100644
index 0000000..46d0586
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/DetailQueryExecutor.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.executor.impl;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.core.scan.result.BatchResult;
+import org.apache.carbondata.core.scan.result.iterator.DetailQueryResultIterator;
+
+/**
+ * Below class will be used to execute the detail query
+ * For executing the detail query it will pass all the block execution
+ * info to detail query result iterator and iterator will be returned
+ */
+public class DetailQueryExecutor extends AbstractQueryExecutor<BatchResult> {
+
+ @Override
+ public CarbonIterator<BatchResult> execute(QueryModel queryModel)
+ throws QueryExecutionException, IOException {
+ List<BlockExecutionInfo> blockExecutionInfoList = getBlockExecutionInfos(queryModel);
+ return new DetailQueryResultIterator(
+ blockExecutionInfoList,
+ queryModel,
+ queryProperties.executorService
+ );
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/QueryExecutorProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/QueryExecutorProperties.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/QueryExecutorProperties.java
new file mode 100644
index 0000000..b4c72e7
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/QueryExecutorProperties.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.executor.impl;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+
+import org.apache.carbondata.core.cache.dictionary.Dictionary;
+import org.apache.carbondata.core.datastore.block.AbstractIndex;
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
+import org.apache.carbondata.core.stats.QueryStatisticsRecorder;
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+
+/**
+ * Holds all the properties required for query execution
+ */
+public class QueryExecutorProperties {
+
+ /**
+ * holds the information required for updating the order block
+ * dictionary key
+ */
+ public KeyStructureInfo keyStructureInfo;
+
+ /**
+ * this will hold the information about the dictionary dimension
+ * which to
+ */
+ public Map<String, Dictionary> columnToDictionayMapping;
+
+ /**
+ * Measure datatypes
+ */
+ public DataType[] measureDataTypes;
+ /**
+ * all the complex dimension which is on filter
+ */
+ public Set<CarbonDimension> complexFilterDimension;
+
+ public Set<CarbonMeasure> filterMeasures;
+ /**
+ * to record the query execution details phase wise
+ */
+ public QueryStatisticsRecorder queryStatisticsRecorder;
+ /**
+ * executor service to execute the query
+ */
+ public ExecutorService executorService;
+ /**
+ * list of blocks in which query will be executed
+ */
+ protected List<AbstractIndex> dataBlocks;
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/VectorDetailQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/VectorDetailQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/VectorDetailQueryExecutor.java
new file mode 100644
index 0000000..21dc6dd
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/VectorDetailQueryExecutor.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.executor.impl;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.core.scan.result.iterator.VectorDetailQueryResultIterator;
+
+/**
+ * Below class will be used to execute the detail query and returns columnar vectors.
+ */
+public class VectorDetailQueryExecutor extends AbstractQueryExecutor<Object> {
+
+ @Override
+ public CarbonIterator<Object> execute(QueryModel queryModel)
+ throws QueryExecutionException, IOException {
+ List<BlockExecutionInfo> blockExecutionInfoList = getBlockExecutionInfos(queryModel);
+ return new VectorDetailQueryResultIterator(
+ blockExecutionInfoList,
+ queryModel,
+ queryProperties.executorService
+ );
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/AggregatorInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/AggregatorInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/AggregatorInfo.java
new file mode 100644
index 0000000..854d150
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/AggregatorInfo.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.executor.infos;
+
+import org.apache.carbondata.core.metadata.DataType;
+
+/**
+ * Info class which store all the details
+ * which is required during aggregation
+ */
+public class AggregatorInfo {
+
+ /**
+ * selected query measure ordinal
+ * which will be used to read the measures chunk data
+ * this will be storing the index of the measure in measures chunk
+ */
+ private int[] measureOrdinals;
+
+ /**
+ * This parameter will be used to
+ * check whether particular measure is present
+ * in the table block, if not then its default value will be used
+ */
+ private boolean[] measureExists;
+
+ /**
+ * this default value will be used to when some measure is not present
+ * in the table block, in case of restructuring of the table if user is adding any
+ * measure then in older block that measure wont be present so for measure default value
+ * will be used to aggregate in the older table block query execution
+ */
+ private Object[] defaultValues;
+
+ /**
+ * Datatype of each measure;
+ */
+ private DataType[] measureDataTypes;
+
+ /**
+ * @return the measureOrdinal
+ */
+ public int[] getMeasureOrdinals() {
+ return measureOrdinals;
+ }
+
+ /**
+ * @param measureOrdinal the measureOrdinal to set
+ */
+ public void setMeasureOrdinals(int[] measureOrdinal) {
+ this.measureOrdinals = measureOrdinal;
+ }
+
+ /**
+ * @return the measureExists
+ */
+ public boolean[] getMeasureExists() {
+ return measureExists;
+ }
+
+ /**
+ * @param measureExists the measureExists to set
+ */
+ public void setMeasureExists(boolean[] measureExists) {
+ this.measureExists = measureExists;
+ }
+
+ /**
+ * @return the defaultValues
+ */
+ public Object[] getDefaultValues() {
+ return defaultValues;
+ }
+
+ /**
+ * @param defaultValues the defaultValues to set
+ */
+ public void setDefaultValues(Object[] defaultValues) {
+ this.defaultValues = defaultValues;
+ }
+
+ public DataType[] getMeasureDataTypes() {
+ return measureDataTypes;
+ }
+
+ public void setMeasureDataTypes(DataType[] measureDataTypes) {
+ this.measureDataTypes = measureDataTypes;
+ }
+}
[53/57] [abbrv] incubator-carbondata git commit: fix style
Posted by ja...@apache.org.
fix style
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/48316190
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/48316190
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/48316190
Branch: refs/heads/movecorepackage
Commit: 4831619042155b9988f103f3bca0519229c3d8f3
Parents: ae17158
Author: jackylk <ja...@huawei.com>
Authored: Fri Jan 13 16:13:46 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Fri Jan 13 16:13:46 2017 +0800
----------------------------------------------------------------------
.../core/atomic/AtomicFileOperationsImpl.java | 6 +-
.../dictionary/AbstractDictionaryCache.java | 4 +-
.../core/compression/BigDecimalCompressor.java | 2 +-
.../core/compression/BigIntCompressor.java | 2 +-
.../core/compression/DoubleCompressor.java | 2 +-
.../core/compression/ValueCompressor.java | 2 +-
.../carbondata/core/datastorage/FileHolder.java | 88 ----
.../core/datastorage/MeasureDataWrapper.java | 30 --
.../core/datastorage/NodeMeasureDataStore.java | 34 --
.../columnar/BlockIndexerStorageForInt.java | 226 ---------
.../BlockIndexerStorageForNoInvertedIndex.java | 159 ------
.../datastorage/columnar/ColumnGroupModel.java | 116 -----
.../columnar/ColumnWithIntIndex.java | 82 ----
.../columnar/ColumnWithIntIndexForHighCard.java | 49 --
.../columnar/ColumnarKeyStoreDataHolder.java | 54 ---
.../columnar/ColumnarKeyStoreMetadata.java | 53 --
.../core/datastorage/columnar/IndexStorage.java | 44 --
.../datastorage/columnar/UnBlockIndexer.java | 78 ---
.../datastorage/compression/Compressor.java | 60 ---
.../compression/CompressorFactory.java | 52 --
.../compression/MeasureMetaDataModel.java | 128 -----
.../compression/ReaderCompressModel.java | 61 ---
.../compression/SnappyCompressor.java | 222 ---------
.../compression/ValueCompressionHolder.java | 115 -----
.../compression/WriterCompressModel.java | 220 ---------
.../decimal/CompressionMaxMinByte.java | 115 -----
.../decimal/CompressionMaxMinDefault.java | 117 -----
.../decimal/CompressionMaxMinInt.java | 113 -----
.../decimal/CompressionMaxMinLong.java | 113 -----
.../decimal/CompressionMaxMinShort.java | 115 -----
.../nondecimal/CompressionNonDecimalByte.java | 99 ----
.../CompressionNonDecimalDefault.java | 101 ----
.../nondecimal/CompressionNonDecimalInt.java | 100 ----
.../nondecimal/CompressionNonDecimalLong.java | 101 ----
.../CompressionNonDecimalMaxMinByte.java | 105 ----
.../CompressionNonDecimalMaxMinDefault.java | 107 ----
.../CompressionNonDecimalMaxMinInt.java | 105 ----
.../CompressionNonDecimalMaxMinLong.java | 106 ----
.../CompressionNonDecimalMaxMinShort.java | 104 ----
.../nondecimal/CompressionNonDecimalShort.java | 100 ----
.../compression/none/CompressionNoneByte.java | 104 ----
.../none/CompressionNoneDefault.java | 102 ----
.../compression/none/CompressionNoneInt.java | 102 ----
.../compression/none/CompressionNoneLong.java | 99 ----
.../compression/none/CompressionNoneShort.java | 104 ----
.../compression/type/CompressionBigDecimal.java | 146 ------
.../dataholder/CarbonReadDataHolder.java | 50 --
.../dataholder/CarbonWriteDataHolder.java | 280 -----------
.../filesystem/AbstractDFSCarbonFile.java | 217 ---------
.../filesystem/AlluxioCarbonFile.java | 130 -----
.../core/datastorage/filesystem/CarbonFile.java | 66 ---
.../filesystem/CarbonFileFilter.java | 24 -
.../datastorage/filesystem/HDFSCarbonFile.java | 128 -----
.../datastorage/filesystem/LocalCarbonFile.java | 229 ---------
.../filesystem/ViewFSCarbonFile.java | 126 -----
.../impl/CompressedDataMeasureDataWrapper.java | 37 --
.../datastorage/impl/DFSFileHolderImpl.java | 134 -----
.../core/datastorage/impl/FileFactory.java | 485 -------------------
.../core/datastorage/impl/FileHolderImpl.java | 198 --------
...ractHeavyCompressedDoubleArrayDataStore.java | 90 ----
...yCompressedDoubleArrayDataInMemoryStore.java | 30 --
.../carbondata/core/datastore/DataRefNode.java | 1 -
.../carbondata/core/datastore/FileHolder.java | 88 ++++
.../core/datastore/MeasureDataWrapper.java | 30 ++
.../core/datastore/NodeMeasureDataStore.java | 34 ++
.../core/datastore/block/SegmentProperties.java | 5 +-
.../core/datastore/block/TableBlockInfo.java | 2 +-
.../datastore/chunk/MeasureColumnDataChunk.java | 2 +-
.../impl/FixedLengthDimensionDataChunk.java | 3 +-
.../impl/VariableLengthDimensionDataChunk.java | 3 +-
.../chunk/reader/CarbonDataReaderFactory.java | 12 +-
.../reader/DimensionColumnChunkReader.java | 2 +-
.../chunk/reader/MeasureColumnChunkReader.java | 2 +-
.../reader/dimension/AbstractChunkReader.java | 4 +-
...mpressedDimensionChunkFileBasedReaderV1.java | 4 +-
...mpressedDimensionChunkFileBasedReaderV2.java | 4 +-
...CompressedMeasureChunkFileBasedReaderV1.java | 8 +-
...CompressedMeasureChunkFileBasedReaderV2.java | 10 +-
.../chunk/store/DimensionChunkStoreFactory.java | 12 +-
.../columnar/BlockIndexerStorageForInt.java | 226 +++++++++
.../BlockIndexerStorageForNoInvertedIndex.java | 159 ++++++
.../datastore/columnar/ColumnGroupModel.java | 116 +++++
.../datastore/columnar/ColumnWithIntIndex.java | 82 ++++
.../columnar/ColumnWithIntIndexForHighCard.java | 49 ++
.../columnar/ColumnarKeyStoreDataHolder.java | 54 +++
.../columnar/ColumnarKeyStoreMetadata.java | 53 ++
.../core/datastore/columnar/IndexStorage.java | 44 ++
.../core/datastore/columnar/UnBlockIndexer.java | 78 +++
.../core/datastore/compression/Compressor.java | 60 +++
.../compression/CompressorFactory.java | 52 ++
.../compression/MeasureMetaDataModel.java | 128 +++++
.../compression/ReaderCompressModel.java | 61 +++
.../datastore/compression/SnappyCompressor.java | 222 +++++++++
.../compression/ValueCompressionHolder.java | 115 +++++
.../compression/WriterCompressModel.java | 220 +++++++++
.../decimal/CompressionMaxMinByte.java | 115 +++++
.../decimal/CompressionMaxMinDefault.java | 117 +++++
.../decimal/CompressionMaxMinInt.java | 113 +++++
.../decimal/CompressionMaxMinLong.java | 113 +++++
.../decimal/CompressionMaxMinShort.java | 115 +++++
.../nondecimal/CompressionNonDecimalByte.java | 99 ++++
.../CompressionNonDecimalDefault.java | 101 ++++
.../nondecimal/CompressionNonDecimalInt.java | 100 ++++
.../nondecimal/CompressionNonDecimalLong.java | 101 ++++
.../CompressionNonDecimalMaxMinByte.java | 105 ++++
.../CompressionNonDecimalMaxMinDefault.java | 107 ++++
.../CompressionNonDecimalMaxMinInt.java | 105 ++++
.../CompressionNonDecimalMaxMinLong.java | 106 ++++
.../CompressionNonDecimalMaxMinShort.java | 104 ++++
.../nondecimal/CompressionNonDecimalShort.java | 100 ++++
.../compression/none/CompressionNoneByte.java | 104 ++++
.../none/CompressionNoneDefault.java | 102 ++++
.../compression/none/CompressionNoneInt.java | 102 ++++
.../compression/none/CompressionNoneLong.java | 99 ++++
.../compression/none/CompressionNoneShort.java | 104 ++++
.../compression/type/CompressionBigDecimal.java | 146 ++++++
.../dataholder/CarbonReadDataHolder.java | 50 ++
.../dataholder/CarbonWriteDataHolder.java | 280 +++++++++++
.../filesystem/AbstractDFSCarbonFile.java | 217 +++++++++
.../datastore/filesystem/AlluxioCarbonFile.java | 130 +++++
.../core/datastore/filesystem/CarbonFile.java | 66 +++
.../datastore/filesystem/CarbonFileFilter.java | 24 +
.../datastore/filesystem/HDFSCarbonFile.java | 128 +++++
.../datastore/filesystem/LocalCarbonFile.java | 229 +++++++++
.../datastore/filesystem/ViewFSCarbonFile.java | 126 +++++
.../impl/CompressedDataMeasureDataWrapper.java | 37 ++
.../core/datastore/impl/DFSFileHolderImpl.java | 134 +++++
.../core/datastore/impl/FileFactory.java | 485 +++++++++++++++++++
.../core/datastore/impl/FileHolderImpl.java | 198 ++++++++
.../impl/btree/AbstractBTreeLeafNode.java | 2 +-
.../datastore/impl/btree/BTreeNonLeafNode.java | 2 +-
.../impl/btree/BlockletBTreeLeafNode.java | 2 +-
...ractHeavyCompressedDoubleArrayDataStore.java | 90 ++++
...yCompressedDoubleArrayDataInMemoryStore.java | 30 ++
.../DirectDictionaryKeyGeneratorFactory.java | 6 +-
.../core/metadata/AbsoluteTableIdentifier.java | 2 +-
.../core/metadata/BlockletInfoColumnar.java | 2 +-
.../core/metadata/ColumnIdentifier.java | 112 +++++
.../carbondata/core/path/CarbonTablePath.java | 11 +-
.../reader/CarbonDeleteDeltaFileReaderImpl.java | 2 +-
.../reader/CarbonDeleteFilesDataReader.java | 2 +-
.../carbondata/core/reader/ThriftReader.java | 2 +-
.../CarbonDictionarySortIndexReaderImpl.java | 2 +-
.../resolver/ConditionalFilterResolverImpl.java | 3 +-
.../processor/AbstractDataBlockIterator.java | 2 +-
.../core/scan/processor/BlocksChunkHolder.java | 2 +-
.../processor/impl/DataBlockIteratorImpl.java | 2 +-
.../AbstractDetailQueryResultIterator.java | 4 +-
.../core/scan/scanner/impl/FilterScanner.java | 2 +-
.../DriverQueryStatisticsRecorderImpl.java | 2 +-
.../core/update/CarbonUpdateUtil.java | 9 +-
.../core/updatestatus/SegmentStatusManager.java | 2 +-
.../SegmentUpdateStatusManager.java | 6 +-
.../core/updatestatus/locks/HdfsFileLock.java | 4 +-
.../core/updatestatus/locks/LocalFileLock.java | 2 +-
.../core/util/CarbonMetadataUtil.java | 4 +-
.../apache/carbondata/core/util/CarbonUtil.java | 14 +-
.../core/util/DataFileFooterConverter.java | 4 +-
.../core/util/ValueCompressionUtil.java | 50 +-
.../writer/CarbonDeleteDeltaWriterImpl.java | 2 +-
.../core/writer/CarbonDictionaryWriterImpl.java | 4 +-
.../carbondata/core/writer/ThriftWriter.java | 2 +-
.../CarbonDictionarySortIndexWriterImpl.java | 4 +-
.../dictionary/AbstractDictionaryCacheTest.java | 4 +-
.../CompressedDataMeasureWrapperTest.java | 4 +-
.../datastorage/DFSFileHolderImplUnitTest.java | 2 +-
.../datastorage/FileFactoryImplUnitTest.java | 2 +-
.../datastorage/FileHolderImplUnitTest.java | 2 +-
.../ColumnarKeyStoreDataHolderUnitTest.java | 93 ----
.../filesystem/AlluxioCarbonFileTest.java | 292 -----------
.../filesystem/HDFSCarbonFileTest.java | 400 ---------------
.../filesystem/LocalCarbonFileTest.java | 471 ------------------
.../filesystem/ViewFsCarbonFileTest.java | 311 ------------
...mpressedMeasureChunkFileBasedReaderTest.java | 10 +-
.../ColumnarKeyStoreDataHolderUnitTest.java | 93 ++++
.../filesystem/AlluxioCarbonFileTest.java | 292 +++++++++++
.../filesystem/HDFSCarbonFileTest.java | 400 +++++++++++++++
.../filesystem/LocalCarbonFileTest.java | 471 ++++++++++++++++++
.../filesystem/ViewFsCarbonFileTest.java | 311 ++++++++++++
...CarbonDictionarySortIndexReaderImplTest.java | 4 +-
.../core/util/CarbonMetadataUtilTest.java | 2 +-
.../carbondata/core/util/CarbonUtilTest.java | 8 +-
.../core/util/DataFileFooterConverterTest.java | 6 +-
.../core/util/ValueCompressionUtilTest.java | 46 +-
.../writer/CarbonDictionaryWriterImplTest.java | 4 +-
.../core/writer/CarbonFooterWriterTest.java | 6 +-
.../carbondata/examples/AlluxioExample.scala | 2 +-
.../examples/GenerateDictionaryExample.scala | 4 +-
.../examples/util/AllDictionaryUtil.scala | 2 +-
.../carbondata/hadoop/CarbonInputFormat.java | 35 +-
.../carbondata/hadoop/CarbonInputSplit.java | 4 +-
.../carbondata/hadoop/CarbonRecordReader.java | 4 +-
.../hadoop/api/CarbonTableInputFormat.java | 4 +-
.../internal/index/impl/InMemoryBTreeIndex.java | 10 +-
.../internal/segment/impl/IndexedSegment.java | 2 +-
.../hadoop/util/BlockLevelTraverser.java | 4 +-
.../hadoop/util/CarbonInputFormatUtil.java | 2 +-
.../carbondata/hadoop/util/SchemaReader.java | 2 +-
.../hadoop/test/util/StoreCreator.java | 2 +-
.../dataload/TestLoadDataGeneral.scala | 2 +-
.../blockprune/BlockPruneQueryTestCase.scala | 2 +-
.../SubqueryWithFilterAndSortTestCase.scala | 2 +-
.../ValueCompressionDataTypeTestCase.scala | 2 +-
.../carbondata/spark/load/CarbonLoaderUtil.java | 42 +-
.../spark/load/DeleteLoadFolders.java | 10 +-
.../spark/merger/CarbonCompactionExecutor.java | 4 +-
.../spark/merger/CarbonCompactionUtil.java | 6 +-
.../spark/merger/CarbonDataMergerUtil.java | 24 +-
.../spark/merger/RowResultMerger.java | 19 +-
.../spark/merger/TupleConversionAdapter.java | 2 +-
.../carbondata/spark/util/CarbonQueryUtil.java | 2 +-
.../carbondata/spark/util/LoadMetadataUtil.java | 2 +-
.../org/apache/carbondata/api/CarbonStore.scala | 2 +-
.../carbondata/spark/CarbonSparkFactory.scala | 2 +-
.../spark/DictionaryDetailHelper.scala | 6 +-
.../spark/rdd/CarbonDataLoadRDD.scala | 3 +-
.../spark/rdd/CarbonGlobalDictionaryRDD.scala | 9 +-
.../spark/rdd/CarbonIUDMergerRDD.scala | 1 -
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 4 +-
.../carbondata/spark/rdd/CarbonScanRDD.scala | 4 +-
.../apache/carbondata/spark/rdd/Compactor.scala | 1 -
.../spark/rdd/DataManagementFunc.scala | 2 +-
.../spark/rdd/NewCarbonDataLoadRDD.scala | 5 +-
.../spark/rdd/UpdateCoalescedRDD.scala | 1 -
.../carbondata/spark/util/CarbonScalaUtil.scala | 4 +-
.../spark/util/GlobalDictionaryUtil.scala | 9 +-
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 2 +-
.../execution/command/carbonTableSchema.scala | 5 +-
.../spark/sql/hive/DistributionUtil.scala | 2 +-
.../scala/org/apache/spark/util/FileUtils.scala | 4 +-
.../spark/CarbonDataFrameWriter.scala | 2 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 6 +-
.../sql/CarbonDatasourceHadoopRelation.scala | 2 +-
.../spark/sql/CarbonDatasourceRelation.scala | 2 +-
.../spark/sql/CarbonDictionaryDecoder.scala | 2 +-
.../scala/org/apache/spark/sql/CarbonScan.scala | 2 +-
.../sql/execution/command/IUDCommands.scala | 15 +-
.../execution/command/carbonTableSchema.scala | 13 +-
.../apache/spark/sql/hive/CarbonMetastore.scala | 6 +-
.../CompactionSystemLockFeatureTest.scala | 2 +-
...GlobalDictionaryUtilConcurrentTestCase.scala | 2 +-
.../VectorizedCarbonRecordReader.java | 10 +-
.../spark/DictionaryDetailHelper.scala | 4 +-
.../execution/command/carbonTableSchema.scala | 2 +-
.../apache/spark/sql/hive/CarbonMetastore.scala | 6 +-
.../processing/csvload/DataGraphExecuter.java | 10 +-
.../processing/csvload/GraphExecutionUtil.java | 12 +-
.../processing/csvreaderstep/BlockDetails.java | 2 +-
.../csvreaderstep/UnivocityCsvParser.java | 4 +-
.../processing/datatypes/PrimitiveDataType.java | 6 +-
.../graphgenerator/GraphGenerator.java | 6 +-
.../processing/mdkeygen/MDKeyGenStep.java | 9 +-
.../processing/model/CarbonLoadModel.java | 4 +-
.../newflow/DataLoadProcessBuilder.java | 2 +-
.../impl/DictionaryFieldConverterImpl.java | 4 +-
.../DirectDictionaryFieldConverterImpl.java | 2 +-
.../converter/impl/FieldEncoderFactory.java | 2 +-
.../impl/MeasureFieldConverterImpl.java | 2 +-
.../impl/NonDictionaryFieldConverterImpl.java | 2 +-
...arallelReadMergeSorterWithBucketingImpl.java | 2 +-
.../steps/DataConverterProcessorStepImpl.java | 2 +-
...ConverterProcessorWithBucketingStepImpl.java | 6 +-
.../steps/DataWriterProcessorStepImpl.java | 4 +-
.../sortdata/CompressedTempSortFileWriter.java | 2 +-
.../sortandgroupby/sortdata/SortParameters.java | 2 +-
.../store/CarbonFactDataHandlerColumnar.java | 22 +-
.../store/CarbonFactDataHandlerModel.java | 6 +-
.../processing/store/StoreFactory.java | 6 +-
.../store/colgroup/ColGroupBlockStorage.java | 2 +-
.../store/writer/AbstractFactDataWriter.java | 14 +-
.../store/writer/CarbonFactDataWriter.java | 4 +-
.../processing/store/writer/NodeHolder.java | 2 +-
.../writer/v1/CarbonFactDataWriterImplV1.java | 4 +-
.../writer/v2/CarbonFactDataWriterImplV2.java | 2 +-
.../csvbased/BadRecordsLogger.java | 4 +-
.../CarbonCSVBasedDimSurrogateKeyGen.java | 2 +-
.../csvbased/CarbonCSVBasedSeqGenMeta.java | 2 +-
.../csvbased/CarbonCSVBasedSeqGenStep.java | 12 +-
.../FileStoreSurrogateKeyGenForCSV.java | 8 +-
.../util/CarbonDataProcessorUtil.java | 12 +-
.../processing/util/CarbonSchemaParser.java | 8 +-
.../carbondata/test/util/StoreCreator.java | 2 +-
282 files changed, 8480 insertions(+), 8358 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/atomic/AtomicFileOperationsImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/atomic/AtomicFileOperationsImpl.java b/core/src/main/java/org/apache/carbondata/core/atomic/AtomicFileOperationsImpl.java
index 33eeae0..db58b17 100644
--- a/core/src/main/java/org/apache/carbondata/core/atomic/AtomicFileOperationsImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/atomic/AtomicFileOperationsImpl.java
@@ -24,9 +24,9 @@ import java.io.DataOutputStream;
import java.io.IOException;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
-import org.apache.carbondata.core.datastorage.impl.FileFactory.FileType;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory.FileType;
public class AtomicFileOperationsImpl implements AtomicFileOperations {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
index 5fd06b4..e5fc83e 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
@@ -26,8 +26,8 @@ import org.apache.carbondata.core.cache.Cache;
import org.apache.carbondata.core.cache.CacheType;
import org.apache.carbondata.core.cache.CarbonLRUCache;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk;
import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java
index 4da8795..fe8192e 100644
--- a/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java
@@ -18,7 +18,7 @@
*/
package org.apache.carbondata.core.compression;
-import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder;
+import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder;
import org.apache.carbondata.core.util.BigDecimalCompressionFinder;
import org.apache.carbondata.core.util.CompressionFinder;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
index e509c5c..6a5250a 100644
--- a/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
@@ -18,7 +18,7 @@
*/
package org.apache.carbondata.core.compression;
-import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder;
+import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
index a716921..345865c 100644
--- a/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
@@ -20,7 +20,7 @@ package org.apache.carbondata.core.compression;
import java.math.BigDecimal;
-import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder;
+import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
index a2d7b51..a74b886 100644
--- a/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
@@ -18,7 +18,7 @@
*/
package org.apache.carbondata.core.compression;
-import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder;
+import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder;
import org.apache.carbondata.core.util.CompressionFinder;
import org.apache.carbondata.core.util.ValueCompressionUtil.COMPRESSION_TYPE;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/FileHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/FileHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/FileHolder.java
deleted file mode 100644
index 96aa069..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/FileHolder.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage;
-
-import java.io.IOException;
-
-public interface FileHolder {
- /**
- * This method will be used to read the byte array from file based on offset
- * and length(number of bytes) need to read
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @param length number of bytes to be read
- * @return read byte array
- */
- byte[] readByteArray(String filePath, long offset, int length) throws IOException;
-
- /**
- * This method will be used to read the byte array from file based on length(number of bytes)
- *
- * @param filePath fully qualified file path
- * @param length number of bytes to be read
- * @return read byte array
- */
- byte[] readByteArray(String filePath, int length) throws IOException;
-
- /**
- * This method will be used to read int from file from postion(offset), here
- * length will be always 4 bacause int byte size if 4
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @return read int
- */
- int readInt(String filePath, long offset) throws IOException;
-
- /**
- * This method will be used to read long from file from postion(offset), here
- * length will be always 8 bacause int byte size is 8
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @return read long
- */
- long readLong(String filePath, long offset) throws IOException;
-
- /**
- * This method will be used to read int from file from postion(offset), here
- * length will be always 4 bacause int byte size if 4
- *
- * @param filePath fully qualified file path
- * @return read int
- */
- int readInt(String filePath) throws IOException;
-
- /**
- * This method will be used to read long value from file from postion(offset), here
- * length will be always 8 because long byte size if 4
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @return read long
- */
- long readDouble(String filePath, long offset) throws IOException;
-
- /**
- * This method will be used to close all the streams currently present in the cache
- */
- void finish() throws IOException;
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/MeasureDataWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/MeasureDataWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastorage/MeasureDataWrapper.java
deleted file mode 100644
index 6fc6c1d..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/MeasureDataWrapper.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage;
-
-import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder;
-
-/**
- * MeasureDataWrapper, interface.
- */
-public interface MeasureDataWrapper {
- CarbonReadDataHolder[] getValues();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/NodeMeasureDataStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/NodeMeasureDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/NodeMeasureDataStore.java
deleted file mode 100644
index fede06c..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/NodeMeasureDataStore.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage;
-
-import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder;
-
-public interface NodeMeasureDataStore {
- /**
- * This method will be used to get the writable key array.
- * writable measure data array will hold below information:
- * <size of measure data array><measure data array>
- * total length will be 4 bytes for size + measure data array length
- *
- * @return writable array (compressed or normal)
- */
- byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] dataHolderArray);
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForInt.java
deleted file mode 100644
index 0a2a8bd..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForInt.java
+++ /dev/null
@@ -1,226 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.columnar;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.util.ByteUtil;
-
-public class BlockIndexerStorageForInt implements IndexStorage<int[]> {
- private boolean alreadySorted;
-
- private int[] dataAfterComp;
-
- private int[] indexMap;
-
- private byte[][] keyBlock;
-
- private int[] dataIndexMap;
-
- private int totalSize;
-
- public BlockIndexerStorageForInt(byte[][] keyBlock, boolean compressData, boolean isNoDictionary,
- boolean isSortRequired) {
- ColumnWithIntIndex[] columnWithIndexs = createColumnWithIndexArray(keyBlock, isNoDictionary);
- if (isSortRequired) {
- Arrays.sort(columnWithIndexs);
- }
- compressMyOwnWay(extractDataAndReturnIndexes(columnWithIndexs, keyBlock));
- if (compressData) {
- compressDataMyOwnWay(columnWithIndexs);
- }
- }
-
- /**
- * Create an object with each column array and respective index
- *
- * @return
- */
- private ColumnWithIntIndex[] createColumnWithIndexArray(byte[][] keyBlock,
- boolean isNoDictionary) {
- ColumnWithIntIndex[] columnWithIndexs;
- if (isNoDictionary) {
- columnWithIndexs = new ColumnWithIntIndexForHighCard[keyBlock.length];
- for (int i = 0; i < columnWithIndexs.length; i++) {
- columnWithIndexs[i] = new ColumnWithIntIndexForHighCard(keyBlock[i], i);
- }
-
- } else {
- columnWithIndexs = new ColumnWithIntIndex[keyBlock.length];
- for (int i = 0; i < columnWithIndexs.length; i++) {
- columnWithIndexs[i] = new ColumnWithIntIndex(keyBlock[i], i);
- }
- }
-
- return columnWithIndexs;
- }
-
- private int[] extractDataAndReturnIndexes(ColumnWithIntIndex[] columnWithIndexs,
- byte[][] keyBlock) {
- int[] indexes = new int[columnWithIndexs.length];
- for (int i = 0; i < indexes.length; i++) {
- indexes[i] = columnWithIndexs[i].getIndex();
- keyBlock[i] = columnWithIndexs[i].getColumn();
- }
- this.keyBlock = keyBlock;
- return indexes;
- }
-
- /**
- * It compresses depends up on the sequence numbers.
- * [1,2,3,4,6,8,10,11,12,13] is translated to [1,4,6,8,10,13] and [0,6]. In
- * first array the start and end of sequential numbers and second array
- * keeps the indexes of where sequential numbers starts. If there is no
- * sequential numbers then the same array it returns with empty second
- * array.
- *
- * @param indexes
- */
- public void compressMyOwnWay(int[] indexes) {
- List<Integer> list = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
- List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
- int k = 0;
- int i = 1;
- for (; i < indexes.length; i++) {
- if (indexes[i] - indexes[i - 1] == 1) {
- k++;
- } else {
- if (k > 0) {
- map.add((list.size()));
- list.add(indexes[i - k - 1]);
- list.add(indexes[i - 1]);
- } else {
- list.add(indexes[i - 1]);
- }
- k = 0;
- }
- }
- if (k > 0) {
- map.add((list.size()));
- list.add(indexes[i - k - 1]);
- list.add(indexes[i - 1]);
- } else {
- list.add(indexes[i - 1]);
- }
- dataAfterComp = convertToArray(list);
- if (indexes.length == dataAfterComp.length) {
- indexMap = new int[0];
- } else {
- indexMap = convertToArray(map);
- }
- if (dataAfterComp.length == 2 && indexMap.length == 1) {
- alreadySorted = true;
- }
- }
-
- private int[] convertToArray(List<Integer> list) {
- int[] shortArray = new int[list.size()];
- for (int i = 0; i < shortArray.length; i++) {
- shortArray[i] = list.get(i);
- }
- return shortArray;
- }
-
- /**
- * @return the alreadySorted
- */
- public boolean isAlreadySorted() {
- return alreadySorted;
- }
-
- /**
- * @return the dataAfterComp
- */
- public int[] getDataAfterComp() {
- return dataAfterComp;
- }
-
- /**
- * @return the indexMap
- */
- public int[] getIndexMap() {
- return indexMap;
- }
-
- /**
- * @return the keyBlock
- */
- public byte[][] getKeyBlock() {
- return keyBlock;
- }
-
- private void compressDataMyOwnWay(ColumnWithIntIndex[] indexes) {
- byte[] prvKey = indexes[0].getColumn();
- List<ColumnWithIntIndex> list =
- new ArrayList<ColumnWithIntIndex>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
- list.add(indexes[0]);
- int counter = 1;
- int start = 0;
- List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
- for (int i = 1; i < indexes.length; i++) {
- if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, indexes[i].getColumn()) != 0) {
- prvKey = indexes[i].getColumn();
- list.add(indexes[i]);
- map.add(start);
- map.add(counter);
- start += counter;
- counter = 1;
- continue;
- }
- counter++;
- }
- map.add(start);
- map.add(counter);
- this.keyBlock = convertToKeyArray(list);
- if (indexes.length == keyBlock.length) {
- dataIndexMap = new int[0];
- } else {
- dataIndexMap = convertToArray(map);
- }
- }
-
- private byte[][] convertToKeyArray(List<ColumnWithIntIndex> list) {
- byte[][] shortArray = new byte[list.size()][];
- for (int i = 0; i < shortArray.length; i++) {
- shortArray[i] = list.get(i).getColumn();
- totalSize += shortArray[i].length;
- }
- return shortArray;
- }
-
- @Override public int[] getDataIndexMap() {
- return dataIndexMap;
- }
-
- @Override public int getTotalSize() {
- return totalSize;
- }
-
- @Override public byte[] getMin() {
- return keyBlock[0];
- }
-
- @Override public byte[] getMax() {
- return keyBlock[keyBlock.length - 1];
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForNoInvertedIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForNoInvertedIndex.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForNoInvertedIndex.java
deleted file mode 100644
index 939d759..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForNoInvertedIndex.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.columnar;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.List;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.util.ByteUtil;
-
-public class BlockIndexerStorageForNoInvertedIndex implements IndexStorage<int[]> {
- private byte[][] keyBlock;
- private byte[][] sortedBlock;
- private int totalSize;
- private int[] dataIndexMap;
-
- public BlockIndexerStorageForNoInvertedIndex(byte[][] keyBlockInput, boolean compressData,
- boolean isNoDictionary) {
- // without invertedindex but can be RLE
- if (compressData) {
- // with RLE
- byte[] prvKey = keyBlockInput[0];
- List<byte[]> list = new ArrayList<byte[]>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
- list.add(keyBlockInput[0]);
- int counter = 1;
- int start = 0;
- List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
- int length = keyBlockInput.length;
- for(int i = 1; i < length; i++) {
- if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, keyBlockInput[i]) != 0) {
- prvKey = keyBlockInput[i];
- list.add(keyBlockInput[i]);
- map.add(start);
- map.add(counter);
- start += counter;
- counter = 1;
- continue;
- }
- counter++;
- }
- map.add(start);
- map.add(counter);
- this.keyBlock = convertToKeyArray(list);
- if (keyBlockInput.length == this.keyBlock.length) {
- dataIndexMap = new int[0];
- } else {
- dataIndexMap = convertToArray(map);
- }
- } else {
- this.keyBlock = keyBlockInput;
- dataIndexMap = new int[0];
- }
-
- this.sortedBlock = new byte[keyBlock.length][];
- System.arraycopy(keyBlock, 0, sortedBlock, 0, keyBlock.length);
- if (isNoDictionary) {
- Arrays.sort(sortedBlock, new Comparator<byte[]>() {
- @Override
- public int compare(byte[] col1, byte[] col2) {
- return ByteUtil.UnsafeComparer.INSTANCE
- .compareTo(col1, 2, col1.length - 2, col2, 2, col2.length - 2);
- }
- });
- } else {
- Arrays.sort(sortedBlock, new Comparator<byte[]>() {
- @Override
- public int compare(byte[] col1, byte[] col2) {
- return ByteUtil.UnsafeComparer.INSTANCE.compareTo(col1, col2);
- }
- });
- }
-
- }
-
- private int[] convertToArray(List<Integer> list) {
- int[] shortArray = new int[list.size()];
- for(int i = 0; i < shortArray.length; i++) {
- shortArray[i] = list.get(i);
- }
- return shortArray;
- }
-
- private byte[][] convertToKeyArray(List<byte[]> list) {
- byte[][] shortArray = new byte[list.size()][];
- for (int i = 0; i < shortArray.length; i++) {
- shortArray[i] = list.get(i);
- totalSize += shortArray[i].length;
- }
- return shortArray;
- }
-
- @Override
- public int[] getDataIndexMap() {
- return dataIndexMap;
- }
-
- @Override
- public int getTotalSize() {
- return totalSize;
- }
-
- @Override
- public boolean isAlreadySorted() {
- return true;
- }
-
- /**
- * no use
- * @return
- */
- @Override
- public int[] getDataAfterComp() {
- return new int[0];
- }
-
- /**
- * no use
- * @return
- */
- @Override
- public int[] getIndexMap() {
- return new int[0];
- }
-
- /**
- * @return the keyBlock
- */
- public byte[][] getKeyBlock() {
- return keyBlock;
- }
-
- @Override public byte[] getMin() {
- return sortedBlock[0];
- }
-
- @Override public byte[] getMax() {
- return sortedBlock[sortedBlock.length - 1];
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnGroupModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnGroupModel.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnGroupModel.java
deleted file mode 100644
index 7402842..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnGroupModel.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.datastorage.columnar;
-
-public class ColumnGroupModel {
-
- /**
- * number of columns in columnar block
- */
- private int[] columnSplit;
-
- /**
- * total number of columns
- */
- private int noOfColumnsStore;
-
- /**
- * whether given index is columnar or not
- * true: columnar
- * false: row block
- */
- private boolean[] columnarStore;
-
- /**
- * column groups
- * e.g
- * {{0,1,2},3,4,{5,6}}
- */
- private int[][] columnGroups;
-
- /**
- * return columnSplit
- *
- * @return
- */
- public int[] getColumnSplit() {
- return columnSplit;
- }
-
- /**
- * set columnSplit
- *
- * @param split
- */
- public void setColumnSplit(int[] split) {
- this.columnSplit = split;
- }
-
- /**
- * @return no of columnar block
- */
- public int getNoOfColumnStore() {
- return this.noOfColumnsStore;
- }
-
- /**
- * set no of columnar block
- *
- * @param noOfColumnsStore
- */
- public void setNoOfColumnStore(int noOfColumnsStore) {
- this.noOfColumnsStore = noOfColumnsStore;
- }
-
- /**
- * it's an identifier for row block or single column block
- *
- * @param columnarStore
- */
- public void setColumnarStore(boolean[] columnarStore) {
- this.columnarStore = columnarStore;
- }
-
- /**
- * set column groups
- *
- * @param columnGroups
- */
- public void setColumnGroup(int[][] columnGroups) {
- this.columnGroups = columnGroups;
- }
-
- /**
- * check if given column group is columnar
- *
- * @param colGroup
- * @return true if given block is columnar
- */
- public boolean isColumnar(int colGroup) {
- return columnarStore[colGroup];
- }
-
- /**
- * @return columngroups
- */
- public int[][] getColumnGroup() {
- return this.columnGroups;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnWithIntIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnWithIntIndex.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnWithIntIndex.java
deleted file mode 100644
index d569572..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnWithIntIndex.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.columnar;
-
-import java.util.Arrays;
-
-import org.apache.carbondata.core.util.ByteUtil;
-
-public class ColumnWithIntIndex implements Comparable<ColumnWithIntIndex> {
- protected byte[] column;
-
- private int index;
-
- public ColumnWithIntIndex(byte[] column, int index) {
- this.column = column;
- this.index = index;
- }
-
- public ColumnWithIntIndex() {
- }
-
- /**
- * @return the column
- */
- public byte[] getColumn() {
- return column;
- }
-
- /**
- * @param column the column to set
- */
- public void setColumn(byte[] column) {
- this.column = column;
- }
-
- /**
- * @return the index
- */
- public int getIndex() {
- return index;
- }
-
- /**
- * @param index the index to set
- */
- public void setIndex(int index) {
- this.index = index;
- }
-
- @Override public int compareTo(ColumnWithIntIndex o) {
- return ByteUtil.UnsafeComparer.INSTANCE.compareTo(column, o.column);
- }
-
- @Override public boolean equals(Object obj) {
- if(obj == null || getClass() != obj.getClass()) {
- return false;
- }
- ColumnWithIntIndex o = (ColumnWithIntIndex)obj;
- return Arrays.equals(column, o.column) && index == o.index;
- }
-
- @Override public int hashCode() {
- return Arrays.hashCode(column) + index;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnWithIntIndexForHighCard.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnWithIntIndexForHighCard.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnWithIntIndexForHighCard.java
deleted file mode 100644
index 61145f7..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnWithIntIndexForHighCard.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.columnar;
-
-import java.util.Arrays;
-
-import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
-
-public class ColumnWithIntIndexForHighCard extends ColumnWithIntIndex
- implements Comparable<ColumnWithIntIndex> {
-
- public ColumnWithIntIndexForHighCard(byte[] column, int index) {
- super(column, index);
- }
-
- @Override public int compareTo(ColumnWithIntIndex o) {
- return UnsafeComparer.INSTANCE
- .compareTo(column, 2, column.length - 2, o.column, 2, o.column.length - 2);
- }
-
- @Override public boolean equals(Object obj) {
- if(obj == null || getClass() != obj.getClass()) {
- return false;
- }
- ColumnWithIntIndexForHighCard o = (ColumnWithIntIndexForHighCard)obj;
- return Arrays.equals(column, o.column) && getIndex() == o.getIndex();
- }
-
- @Override public int hashCode() {
- return Arrays.hashCode(column) + getIndex();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreDataHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreDataHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreDataHolder.java
deleted file mode 100644
index 31bfd00..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreDataHolder.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.columnar;
-
-import java.nio.ByteBuffer;
-
-public class ColumnarKeyStoreDataHolder {
- private byte[] keyblockData;
- private ColumnarKeyStoreMetadata columnarKeyStoreMetadata;
-
- public ColumnarKeyStoreDataHolder(final byte[] keyblockData,
- final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) {
- this.keyblockData = keyblockData;
- this.columnarKeyStoreMetadata = columnarKeyStoreMetadata;
- }
-
- public ColumnarKeyStoreDataHolder(final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) {
- this.columnarKeyStoreMetadata = columnarKeyStoreMetadata;
- }
-
- public int getSurrogateKey(int columnIndex) {
- byte[] actual = new byte[4];
- int startIndex;
- if (null != columnarKeyStoreMetadata.getColumnReverseIndex()) {
- startIndex =
- columnarKeyStoreMetadata.getColumnReverseIndex()[columnIndex] * columnarKeyStoreMetadata
- .getEachRowSize();
- } else {
- startIndex = columnIndex * columnarKeyStoreMetadata.getEachRowSize();
- }
- int destPos = 4 - columnarKeyStoreMetadata.getEachRowSize();
- System.arraycopy(keyblockData, startIndex, actual, destPos,
- columnarKeyStoreMetadata.getEachRowSize());
- return ByteBuffer.wrap(actual).getInt();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreMetadata.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreMetadata.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreMetadata.java
deleted file mode 100644
index 6badce5..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreMetadata.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.columnar;
-
-class ColumnarKeyStoreMetadata {
-
- private int[] columnReverseIndex;
-
- private int eachRowSize;
-
- ColumnarKeyStoreMetadata(int eachRowSize) {
- this.eachRowSize = eachRowSize;
- }
-
- /**
- * @return the eachRowSize
- */
- int getEachRowSize() {
- return eachRowSize;
- }
-
- /**
- * @return the columnReverseIndex
- */
- int[] getColumnReverseIndex() {
- return columnReverseIndex;
- }
-
- /**
- * @param columnReverseIndex the columnReverseIndex to set
- */
- void setColumnReverseIndex(int[] columnReverseIndex) {
- this.columnReverseIndex = columnReverseIndex;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/IndexStorage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/IndexStorage.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/IndexStorage.java
deleted file mode 100644
index bc8b60e..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/IndexStorage.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.columnar;
-
-public interface IndexStorage<T> {
- boolean isAlreadySorted();
-
- T getDataAfterComp();
-
- T getIndexMap();
-
- byte[][] getKeyBlock();
-
- T getDataIndexMap();
-
- int getTotalSize();
-
- /**
- * @return min value of block
- */
- byte[] getMin();
-
- /**
- * @return max value of block
- */
- byte[] getMax();
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/UnBlockIndexer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/UnBlockIndexer.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/UnBlockIndexer.java
deleted file mode 100644
index 4f2a80e..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/UnBlockIndexer.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.columnar;
-
-import java.util.Arrays;
-
-public final class UnBlockIndexer {
-
- private UnBlockIndexer() {
-
- }
-
- public static int[] uncompressIndex(int[] indexData, int[] indexMap) {
- int actualSize = indexData.length;
- for (int i = 0; i < indexMap.length; i++) {
- actualSize += indexData[indexMap[i] + 1] - indexData[indexMap[i]] - 1;
- }
- int[] indexes = new int[actualSize];
- int k = 0;
- for (int i = 0; i < indexData.length; i++) {
- int index = Arrays.binarySearch(indexMap, i);
- if (index > -1) {
- for (int j = indexData[indexMap[index]]; j <= indexData[indexMap[index] + 1]; j++) {
- indexes[k] = j;
- k++;
- }
- i++;
- } else {
- indexes[k] = indexData[i];
- k++;
- }
- }
- return indexes;
- }
-
- public static byte[] uncompressData(byte[] data, int[] index, int keyLen) {
- if (index.length < 1) {
- return data;
- }
- int numberOfCopy = 0;
- int actualSize = 0;
- int srcPos = 0;
- int destPos = 0;
- for (int i = 1; i < index.length; i += 2) {
- actualSize += index[i];
- }
- byte[] uncompressedData = new byte[actualSize * keyLen];
- int picIndex = 0;
- for (int i = 0; i < data.length; i += keyLen) {
- numberOfCopy = index[picIndex * 2 + 1];
- picIndex++;
- for (int j = 0; j < numberOfCopy; j++) {
- System.arraycopy(data, srcPos, uncompressedData, destPos, keyLen);
- destPos += keyLen;
- }
- srcPos += keyLen;
- }
- return uncompressedData;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/Compressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/Compressor.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/Compressor.java
deleted file mode 100644
index 7982d54..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/Compressor.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.compression;
-
-public interface Compressor {
-
- byte[] compressByte(byte[] unCompInput);
-
- byte[] unCompressByte(byte[] compInput);
-
- byte[] unCompressByte(byte[] compInput, int offset, int length);
-
- byte[] compressShort(short[] unCompInput);
-
- short[] unCompressShort(byte[] compInput);
-
- short[] unCompressShort(byte[] compInput, int offset, int lenght);
-
- byte[] compressInt(int[] unCompInput);
-
- int[] unCompressInt(byte[] compInput);
-
- int[] unCompressInt(byte[] compInput, int offset, int length);
-
- byte[] compressLong(long[] unCompInput);
-
- long[] unCompressLong(byte[] compInput);
-
- long[] unCompressLong(byte[] compInput, int offset, int length);
-
- byte[] compressFloat(float[] unCompInput);
-
- float[] unCompressFloat(byte[] compInput);
-
- float[] unCompressFloat(byte[] compInput, int offset, int length);
-
- byte[] compressDouble(double[] unCompInput);
-
- double[] unCompressDouble(byte[] compInput);
-
- double[] unCompressDouble(byte[] compInput, int offset, int length);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/CompressorFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/CompressorFactory.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/CompressorFactory.java
deleted file mode 100644
index 67c24d1..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/CompressorFactory.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.compression;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.util.CarbonProperties;
-
-public class CompressorFactory {
-
- private static final CompressorFactory COMPRESSOR_FACTORY = new CompressorFactory();
-
- private final Compressor compressor;
-
- private CompressorFactory() {
- String compressorType = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.COMPRESSOR, CarbonCommonConstants.DEFAULT_COMPRESSOR);
- switch (compressorType) {
- case "snappy":
- compressor = new SnappyCompressor();
- break;
- default:
- throw new RuntimeException(
- "Invalid compressor type provided! Please provide valid compressor type");
- }
- }
-
- public static CompressorFactory getInstance() {
- return COMPRESSOR_FACTORY;
- }
-
- public Compressor getCompressor() {
- return compressor;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/MeasureMetaDataModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/MeasureMetaDataModel.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/MeasureMetaDataModel.java
deleted file mode 100644
index 41a1ef0..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/MeasureMetaDataModel.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.compression;
-
-public class MeasureMetaDataModel {
- /**
- * maxValue
- */
- private Object[] maxValue;
-
- /**
- * minValue
- */
- private Object[] minValue;
-
- /**
- * mantissa
- */
- private int[] mantissa;
-
- /**
- * measureCount
- */
- private int measureCount;
-
- /**
- * uniqueValue
- */
- private Object[] uniqueValue;
-
- /**
- * type
- */
- private char[] type;
-
- /**
- * dataTypeSelected
- */
- private byte[] dataTypeSelected;
-
- public MeasureMetaDataModel(Object[] minValue, Object[] maxValue, int[] mantissa,
- int measureCount, Object[] uniqueValue, char[] type, byte[] dataTypeSelected) {
- this.minValue = minValue;
- this.maxValue = maxValue;
- this.mantissa = mantissa;
- this.measureCount = measureCount;
- this.uniqueValue = uniqueValue;
- this.type = type;
- this.dataTypeSelected = dataTypeSelected;
- }
-
- /**
- * get Max value
- *
- * @return
- */
- public Object[] getMaxValue() {
- return maxValue;
- }
-
- /**
- * getMinValue
- *
- * @return
- */
- public Object[] getMinValue() {
- return minValue;
- }
-
- /**
- * getMantissa
- *
- * @return
- */
- public int[] getMantissa() {
- return mantissa;
- }
-
- /**
- * getMeasureCount
- *
- * @return
- */
- public int getMeasureCount() {
- return measureCount;
- }
-
- /**
- * getUniqueValue
- *
- * @return
- */
- public Object[] getUniqueValue() {
- return uniqueValue;
- }
-
- /**
- * @return the type
- */
- public char[] getType() {
- return type;
- }
-
- /**
- * @return the dataTypeSelected
- */
- public byte[] getDataTypeSelected() {
- return dataTypeSelected;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/ReaderCompressModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/ReaderCompressModel.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/ReaderCompressModel.java
deleted file mode 100644
index 02da265..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/ReaderCompressModel.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.compression;
-
-import org.apache.carbondata.core.metadata.ValueEncoderMeta;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-
-// Used in read path for decompression preparation
-public class ReaderCompressModel {
- private ValueEncoderMeta valueEncoderMeta;
-
- private ValueCompressionUtil.DataType convertedDataType;
-
- private ValueCompressionHolder valueHolder;
-
- public void setValueEncoderMeta(ValueEncoderMeta valueEncoderMeta) {
- this.valueEncoderMeta = valueEncoderMeta;
- }
-
- public ValueCompressionUtil.DataType getConvertedDataType() {
- return convertedDataType;
- }
-
- public void setConvertedDataType(ValueCompressionUtil.DataType convertedDataType) {
- this.convertedDataType = convertedDataType;
- }
-
- public Object getMaxValue() {
- return valueEncoderMeta.getMaxValue();
- }
-
- public int getMantissa() {
- return valueEncoderMeta.getMantissa();
- }
-
- public ValueCompressionHolder getValueCompressionHolder() {
- return valueHolder;
- }
-
- public void setValueCompressionHolder(ValueCompressionHolder valueHolder) {
- this.valueHolder = valueHolder;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/SnappyCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/SnappyCompressor.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/SnappyCompressor.java
deleted file mode 100644
index 0ca6512..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/SnappyCompressor.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.compression;
-
-import java.io.IOException;
-import java.lang.reflect.Field;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-
-import org.xerial.snappy.Snappy;
-import org.xerial.snappy.SnappyNative;
-
-public class SnappyCompressor implements Compressor {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(SnappyCompressor.class.getName());
-
- private final SnappyNative snappyNative;
-
- public SnappyCompressor() {
- Snappy snappy = new Snappy();
- Field privateField = null;
- try {
- privateField = snappy.getClass().getDeclaredField("impl");
- } catch (NoSuchFieldException | SecurityException e) {
- throw new RuntimeException(e);
- }
- privateField.setAccessible(true);
- try {
- snappyNative = (SnappyNative) privateField.get(snappy);
- } catch (IllegalArgumentException | IllegalAccessException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override public byte[] compressByte(byte[] unCompInput) {
- try {
- return Snappy.rawCompress(unCompInput, unCompInput.length);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- return null;
- }
- }
-
- @Override public byte[] unCompressByte(byte[] compInput) {
- try {
- return Snappy.uncompress(compInput);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
- return compInput;
- }
-
- @Override public byte[] unCompressByte(byte[] compInput, int offset, int length) {
- int uncompressedLength = 0;
- byte[] data = null;
- try {
- uncompressedLength = Snappy.uncompressedLength(compInput, offset, length);
- data = new byte[uncompressedLength];
- Snappy.uncompress(compInput, offset, length, data, 0);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
- return data;
- }
-
- @Override public byte[] compressShort(short[] unCompInput) {
- try {
- return Snappy.compress(unCompInput);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- return null;
- }
- }
-
- @Override public short[] unCompressShort(byte[] compInput) {
- try {
- return Snappy.uncompressShortArray(compInput);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
- return null;
- }
-
- @Override public short[] unCompressShort(byte[] compInput, int offset, int lenght) {
- try {
- return Snappy.uncompressShortArray(compInput, offset, lenght);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
- return null;
- }
-
- @Override public byte[] compressInt(int[] unCompInput) {
- try {
- return Snappy.compress(unCompInput);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- return null;
- }
- }
-
- @Override public int[] unCompressInt(byte[] compInput) {
- try {
- return Snappy.uncompressIntArray(compInput);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
- return null;
- }
-
- @Override public int[] unCompressInt(byte[] compInput, int offset, int length) {
- try {
- return Snappy.uncompressIntArray(compInput, offset, length);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
- return null;
- }
-
- @Override public byte[] compressLong(long[] unCompInput) {
- try {
- return Snappy.compress(unCompInput);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- return null;
- }
- }
-
- @Override public long[] unCompressLong(byte[] compInput) {
- try {
- return Snappy.uncompressLongArray(compInput);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
- return null;
- }
-
- @Override public long[] unCompressLong(byte[] compInput, int offset, int length) {
- try {
- return Snappy.uncompressLongArray(compInput, offset, length);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
- return null;
- }
-
- @Override public byte[] compressFloat(float[] unCompInput) {
- try {
- return Snappy.compress(unCompInput);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- return null;
- }
- }
-
- @Override public float[] unCompressFloat(byte[] compInput) {
- try {
- return Snappy.uncompressFloatArray(compInput);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
- return null;
- }
-
- @Override public float[] unCompressFloat(byte[] compInput, int offset, int length) {
- try {
- return Snappy.uncompressFloatArray(compInput, offset, length);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
- return null;
- }
-
- @Override public byte[] compressDouble(double[] unCompInput) {
- try {
- return Snappy.compress(unCompInput);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- return null;
- }
- }
-
- @Override public double[] unCompressDouble(byte[] compInput) {
- try {
- return Snappy.uncompressDoubleArray(compInput);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
- return null;
- }
-
- @Override public double[] unCompressDouble(byte[] compInput, int offset, int length) {
- try {
- int uncompressedLength = Snappy.uncompressedLength(compInput, offset, length);
- double[] result = new double[uncompressedLength / 8];
- snappyNative.rawUncompress(compInput, offset, length, result, 0);
- return result;
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/ValueCompressionHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/ValueCompressionHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/ValueCompressionHolder.java
deleted file mode 100644
index 425c266..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/ValueCompressionHolder.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.compression;
-
-import java.math.BigDecimal;
-
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-
-
-/**
- * ValueCompressionHolder is the base class for handling
- * compression / decompression of the measure data chunk
- */
-public abstract class ValueCompressionHolder<T> {
-
- /**
- * compressedValue
- */
- protected byte[] compressedValue;
-
- /**
- * @param compressor the compressor used to decompress the data
- * @param dataType data type of the data
- * @param data compressed data
- */
- public void unCompress(Compressor compressor, DataType dataType, byte[] data,
- int offset, int length) {
- switch (dataType) {
- case DATA_BYTE:
- setValue((T)compressor.unCompressByte(data, offset, length));
- break;
- case DATA_SHORT:
- setValue((T)compressor.unCompressShort(data, offset, length));
- break;
- case DATA_INT:
- setValue((T)compressor.unCompressInt(data, offset, length));
- break;
- case DATA_LONG:
- case DATA_BIGINT:
- setValue((T)compressor.unCompressLong(data, offset, length));
- break;
- case DATA_FLOAT:
- setValue((T)compressor.unCompressFloat(data, offset, length));
- break;
- default:
- setValue((T)compressor.unCompressDouble(data, offset, length));
- break;
- }
- }
-
- /**
- * @param compressor the compressor used to compress the data
- * @param dataType data type of the data
- * @param data original data
- */
- public byte[] compress(Compressor compressor, DataType dataType, Object data) {
- switch (dataType) {
- case DATA_BYTE:
- return compressor.compressByte((byte[])data);
- case DATA_SHORT:
- return compressor.compressShort((short[])data);
- case DATA_INT:
- return compressor.compressInt((int[])data);
- case DATA_LONG:
- case DATA_BIGINT:
- return compressor.compressLong((long[])data);
- case DATA_FLOAT:
- return compressor.compressFloat((float[])data);
- case DATA_DOUBLE:
- default:
- return compressor.compressDouble((double[])data);
- }
- }
-
- public abstract void setValue(T value);
-
- public abstract T getValue();
-
- public abstract void setValueInBytes(byte[] value);
-
- public abstract void compress();
-
- public abstract void uncompress(DataType dataType, byte[] compressData, int offset,
- int length, int decimal, Object maxValueObject);
-
- public byte[] getCompressedData() { return compressedValue; }
-
- public abstract long getLongValue(int index);
-
- public abstract double getDoubleValue(int index);
-
- public abstract BigDecimal getBigDecimalValue(int index);
-
- public abstract void freeMemory();
-
-
-}
[45/57] [abbrv] incubator-carbondata git commit: fix style
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java b/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java
index 0036265..d3434c7 100644
--- a/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java
@@ -30,9 +30,9 @@ import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.BlockletInfoColumnar;
import org.apache.carbondata.core.reader.CarbonFooterReader;
import org.apache.carbondata.core.util.CarbonMetadataUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/examples/spark/src/main/scala/org/apache/carbondata/examples/AlluxioExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/AlluxioExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/AlluxioExample.scala
index 963552e..826819d 100644
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/AlluxioExample.scala
+++ b/examples/spark/src/main/scala/org/apache/carbondata/examples/AlluxioExample.scala
@@ -18,7 +18,7 @@
package org.apache.carbondata.examples
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastorage.impl.FileFactory
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.examples.util.ExampleUtils
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/examples/spark/src/main/scala/org/apache/carbondata/examples/GenerateDictionaryExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/GenerateDictionaryExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/GenerateDictionaryExample.scala
index 1a0e9bc..2016cc2 100644
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/GenerateDictionaryExample.scala
+++ b/examples/spark/src/main/scala/org/apache/carbondata/examples/GenerateDictionaryExample.scala
@@ -20,10 +20,10 @@ package org.apache.carbondata.examples
import org.apache.spark.sql.{CarbonContext, CarbonEnv, CarbonRelation}
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
-import org.apache.carbondata.core.path.CarbonStorePath
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.CarbonTableIdentifier
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
+import org.apache.carbondata.core.path.CarbonStorePath
import org.apache.carbondata.examples.util.ExampleUtils
import org.apache.carbondata.spark.load.CarbonLoaderUtil
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/examples/spark/src/main/scala/org/apache/carbondata/examples/util/AllDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/util/AllDictionaryUtil.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/util/AllDictionaryUtil.scala
index e16e084..3ab61bf 100644
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/util/AllDictionaryUtil.scala
+++ b/examples/spark/src/main/scala/org/apache/carbondata/examples/util/AllDictionaryUtil.scala
@@ -24,7 +24,7 @@ import scala.collection.mutable.{ArrayBuffer, HashSet}
import org.apache.spark.SparkContext
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.datastorage.impl.FileFactory
+import org.apache.carbondata.core.datastore.impl.FileFactory
object AllDictionaryUtil {
private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
index 362510e..fa4f3f2 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
@@ -20,27 +20,42 @@ package org.apache.carbondata.hadoop;
import java.io.IOException;
import java.lang.reflect.Constructor;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
-import org.apache.carbondata.core.update.data.BlockMappingVO;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.datastore.DataRefNodeFinder;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.SegmentTaskIndexStore;
import org.apache.carbondata.core.datastore.TableSegmentUniqueIdentifier;
-import org.apache.carbondata.core.datastore.block.*;
+import org.apache.carbondata.core.datastore.block.AbstractIndex;
+import org.apache.carbondata.core.datastore.block.BlockletInfos;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.block.SegmentTaskIndexWrapper;
+import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.impl.btree.BTreeDataRefNodeFinder;
import org.apache.carbondata.core.datastore.impl.btree.BlockBTreeLeafNode;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.filter.FilterExpressionProcessor;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+import org.apache.carbondata.core.scan.model.CarbonQueryPlan;
+import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.core.stats.QueryStatistic;
import org.apache.carbondata.core.stats.QueryStatisticsConstants;
import org.apache.carbondata.core.stats.QueryStatisticsRecorder;
+import org.apache.carbondata.core.update.data.BlockMappingVO;
import org.apache.carbondata.core.update.data.UpdateVO;
import org.apache.carbondata.core.updatestatus.SegmentStatusManager;
import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager;
@@ -52,12 +67,6 @@ import org.apache.carbondata.hadoop.util.BlockLevelTraverser;
import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil;
import org.apache.carbondata.hadoop.util.ObjectSerializationUtil;
import org.apache.carbondata.hadoop.util.SchemaReader;
-import org.apache.carbondata.core.scan.expression.Expression;
-import org.apache.carbondata.core.scan.filter.FilterExpressionProcessor;
-import org.apache.carbondata.core.scan.filter.FilterUtil;
-import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
-import org.apache.carbondata.core.scan.model.CarbonQueryPlan;
-import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
index 115fc7d..d5f717b 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
@@ -27,12 +27,12 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.BlockletInfos;
import org.apache.carbondata.core.datastore.block.Distributable;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.path.CarbonTablePath;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.update.data.UpdateVO;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.hadoop.internal.index.Block;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
index af6611f..94aeddf 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
@@ -26,13 +26,13 @@ import java.util.Map;
import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.hadoop.readsupport.CarbonReadSupport;
import org.apache.carbondata.core.scan.executor.QueryExecutor;
import org.apache.carbondata.core.scan.executor.QueryExecutorFactory;
import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.core.scan.result.iterator.ChunkRowIterator;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.hadoop.readsupport.CarbonReadSupport;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index 0eca474..b2cf663 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -23,6 +23,8 @@ import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.carbondata.hadoop.CarbonProjection;
import org.apache.carbondata.hadoop.internal.CarbonInputSplit;
import org.apache.carbondata.hadoop.internal.segment.Segment;
@@ -30,8 +32,6 @@ import org.apache.carbondata.hadoop.internal.segment.SegmentManager;
import org.apache.carbondata.hadoop.internal.segment.SegmentManagerFactory;
import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil;
import org.apache.carbondata.hadoop.util.ObjectSerializationUtil;
-import org.apache.carbondata.core.scan.expression.Expression;
-import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.InputSplit;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
index 221700c..3d71b89 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
@@ -26,7 +26,6 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.datastore.DataRefNodeFinder;
import org.apache.carbondata.core.datastore.IndexKey;
@@ -39,19 +38,20 @@ import org.apache.carbondata.core.datastore.block.SegmentTaskIndexWrapper;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.impl.btree.BTreeDataRefNodeFinder;
import org.apache.carbondata.core.datastore.impl.btree.BlockBTreeLeafNode;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.scan.filter.FilterExpressionProcessor;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.carbondata.core.stats.QueryStatistic;
import org.apache.carbondata.core.stats.QueryStatisticsConstants;
import org.apache.carbondata.core.stats.QueryStatisticsRecorder;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
import org.apache.carbondata.hadoop.CacheClient;
import org.apache.carbondata.hadoop.CarbonInputSplit;
import org.apache.carbondata.hadoop.internal.index.Block;
import org.apache.carbondata.hadoop.internal.index.Index;
import org.apache.carbondata.hadoop.internal.segment.Segment;
-import org.apache.carbondata.core.scan.filter.FilterExpressionProcessor;
-import org.apache.carbondata.core.scan.filter.FilterUtil;
-import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/impl/IndexedSegment.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/impl/IndexedSegment.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/impl/IndexedSegment.java
index 0215e26..f49cb33 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/impl/IndexedSegment.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/impl/IndexedSegment.java
@@ -23,11 +23,11 @@ import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.carbondata.hadoop.internal.index.Block;
import org.apache.carbondata.hadoop.internal.index.Index;
import org.apache.carbondata.hadoop.internal.index.IndexLoader;
import org.apache.carbondata.hadoop.internal.segment.Segment;
-import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.JobContext;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
index 1800fc3..9918bad 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
@@ -25,12 +25,12 @@ import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.impl.btree.BTreeDataRefNodeFinder;
import org.apache.carbondata.core.datastore.impl.btree.BlockBTreeLeafNode;
-import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.path.CarbonTablePath;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.update.CarbonUpdateUtil;
import org.apache.carbondata.core.update.SegmentUpdateDetails;
import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager;
-import org.apache.carbondata.core.scan.filter.FilterUtil;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
index 790a42c..c8f9496 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
@@ -26,7 +26,6 @@ import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-import org.apache.carbondata.hadoop.CarbonInputFormat;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.filter.FilterExpressionProcessor;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
@@ -34,6 +33,7 @@ import org.apache.carbondata.core.scan.model.CarbonQueryPlan;
import org.apache.carbondata.core.scan.model.QueryDimension;
import org.apache.carbondata.core.scan.model.QueryMeasure;
import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.hadoop.CarbonInputFormat;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Job;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
index 4450339..4e4caa0 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
@@ -20,6 +20,7 @@ package org.apache.carbondata.hadoop.util;
import java.io.IOException;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.converter.SchemaConverter;
@@ -28,7 +29,6 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.TableInfo;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.reader.ThriftReader;
import org.apache.thrift.TBase;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
index 9a6e269..91e6d11 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
@@ -47,7 +47,7 @@ import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.processing.csvreaderstep.BlockDetails;
import org.apache.carbondata.core.updatestatus.LoadMetadataDetails;
import org.apache.carbondata.core.util.CarbonProperties;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
index 3555477..8528e44 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
@@ -27,7 +27,7 @@ import org.apache.spark.sql.Row
import org.apache.spark.sql.common.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.datastorage.impl.FileFactory
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.path.{CarbonStorePath, CarbonTablePath}
class TestLoadDataGeneral extends QueryTest with BeforeAndAfterAll {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/blockprune/BlockPruneQueryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/blockprune/BlockPruneQueryTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/blockprune/BlockPruneQueryTestCase.scala
index 18efde8..4e6f685 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/blockprune/BlockPruneQueryTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/blockprune/BlockPruneQueryTestCase.scala
@@ -24,7 +24,7 @@ import org.apache.spark.sql.Row
import org.apache.spark.sql.common.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.datastorage.impl.FileFactory
+import org.apache.carbondata.core.datastore.impl.FileFactory
/**
* This class contains test cases for block prune query
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SubqueryWithFilterAndSortTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SubqueryWithFilterAndSortTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SubqueryWithFilterAndSortTestCase.scala
index 9b4ac04..05f954b 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SubqueryWithFilterAndSortTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SubqueryWithFilterAndSortTestCase.scala
@@ -22,7 +22,7 @@ package org.apache.carbondata.spark.testsuite.detailquery
import org.apache.spark.sql.common.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.datastorage.impl.FileFactory
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.datastorage.store.impl.FileFactory.FileType
class SubqueryWithFilterAndSortTestCase extends QueryTest with BeforeAndAfterAll {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ValueCompressionDataTypeTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ValueCompressionDataTypeTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ValueCompressionDataTypeTestCase.scala
index 81a2b00..3b16baf 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ValueCompressionDataTypeTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ValueCompressionDataTypeTestCase.scala
@@ -21,7 +21,7 @@ package org.apache.carbondata.spark.testsuite.detailquery
import org.apache.spark.sql.common.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.datastorage.impl.FileFactory
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.datastorage.store.impl.FileFactory.FileType
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
index 75bc294..0f783e8 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
@@ -18,43 +18,57 @@
*/
package org.apache.carbondata.spark.load;
-import java.io.*;
+import java.io.BufferedWriter;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.nio.charset.Charset;
import java.text.SimpleDateFormat;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.atomic.AtomicFileOperations;
+import org.apache.carbondata.core.atomic.AtomicFileOperationsImpl;
+import org.apache.carbondata.core.atomic.FileWriteOperation;
import org.apache.carbondata.core.cache.Cache;
import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.cache.CacheType;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.block.Distributable;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory.FileType;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnIdentifier;
-import org.apache.carbondata.core.datastore.block.Distributable;
-import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
-import org.apache.carbondata.core.datastorage.impl.FileFactory.FileType;
-import org.apache.carbondata.core.updatestatus.LoadMetadataDetails;
import org.apache.carbondata.core.update.CarbonUpdateUtil;
+import org.apache.carbondata.core.updatestatus.LoadMetadataDetails;
import org.apache.carbondata.core.updatestatus.SegmentStatusManager;
+import org.apache.carbondata.core.updatestatus.locks.ICarbonLock;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.atomic.AtomicFileOperations;
-import org.apache.carbondata.core.atomic.AtomicFileOperationsImpl;
-import org.apache.carbondata.core.atomic.FileWriteOperation;
-import org.apache.carbondata.core.updatestatus.locks.ICarbonLock;
import org.apache.carbondata.processing.api.dataloader.DataLoadModel;
import org.apache.carbondata.processing.api.dataloader.SchemaInfo;
import org.apache.carbondata.processing.csvload.DataGraphExecuter;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java
index 470ab9a..aac004f 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java
@@ -35,15 +35,15 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
-import org.apache.carbondata.core.updatestatus.LoadMetadataDetails;
import org.apache.carbondata.core.update.CarbonUpdateUtil;
+import org.apache.carbondata.core.updatestatus.LoadMetadataDetails;
public final class DeleteLoadFolders {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionExecutor.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionExecutor.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionExecutor.java
index 1314499..8e66682 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionExecutor.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionExecutor.java
@@ -29,6 +29,7 @@ import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.block.TaskBlockInfo;
@@ -36,8 +37,6 @@ import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.scan.executor.QueryExecutor;
import org.apache.carbondata.core.scan.executor.QueryExecutorFactory;
import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
@@ -46,6 +45,7 @@ import org.apache.carbondata.core.scan.model.QueryMeasure;
import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.core.scan.result.BatchResult;
import org.apache.carbondata.core.scan.result.iterator.RawResultIterator;
+import org.apache.carbondata.core.util.CarbonUtil;
/**
* Executor class for executing the query on the selected segments to be merged.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionUtil.java
index fb19387..8147e25 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionUtil.java
@@ -26,14 +26,14 @@ import java.util.Map;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.block.TaskBlockInfo;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.path.CarbonTablePath;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.util.CarbonUtil;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
index 82d3f0e..ca87db9 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
@@ -23,31 +23,39 @@ import java.io.File;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
-import org.apache.carbondata.core.updatestatus.LoadMetadataDetails;
import org.apache.carbondata.core.reader.CarbonDeleteFilesDataReader;
import org.apache.carbondata.core.update.CarbonUpdateUtil;
import org.apache.carbondata.core.update.DeleteDeltaBlockDetails;
import org.apache.carbondata.core.update.SegmentUpdateDetails;
+import org.apache.carbondata.core.updatestatus.LoadMetadataDetails;
import org.apache.carbondata.core.updatestatus.SegmentStatusManager;
import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager;
-import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.writer.CarbonDeleteDeltaWriterImpl;
import org.apache.carbondata.core.updatestatus.locks.CarbonLockFactory;
import org.apache.carbondata.core.updatestatus.locks.ICarbonLock;
import org.apache.carbondata.core.updatestatus.locks.LockUsage;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.writer.CarbonDeleteDeltaWriterImpl;
import org.apache.carbondata.processing.model.CarbonLoadModel;
import org.apache.carbondata.spark.load.CarbonLoaderUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/RowResultMerger.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/RowResultMerger.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/RowResultMerger.java
index 3802ed2..7eb3f8f 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/RowResultMerger.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/RowResultMerger.java
@@ -19,21 +19,30 @@
package org.apache.carbondata.spark.merger;
import java.io.File;
-import java.util.*;
+import java.util.AbstractQueue;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.PriorityQueue;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.metadata.CarbonMetadata;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.scan.result.iterator.RawResultIterator;
+import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
import org.apache.carbondata.core.update.CarbonUpdateUtil;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonUtil;
@@ -46,8 +55,6 @@ import org.apache.carbondata.processing.store.CarbonFactDataHandlerColumnar;
import org.apache.carbondata.processing.store.CarbonFactDataHandlerModel;
import org.apache.carbondata.processing.store.CarbonFactHandler;
import org.apache.carbondata.processing.store.writer.exception.CarbonDataWriterException;
-import org.apache.carbondata.core.scan.result.iterator.RawResultIterator;
-import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
/**
* This is the Merger class responsible for the merging of the segments.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/TupleConversionAdapter.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/TupleConversionAdapter.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/TupleConversionAdapter.java
index 497605c..b5f3f7f 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/TupleConversionAdapter.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/TupleConversionAdapter.java
@@ -22,8 +22,8 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.processing.util.RemoveDictionaryUtil;
import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
+import org.apache.carbondata.processing.util.RemoveDictionaryUtil;
/**
* This class will be used to convert the Result into the format used in data writer.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/CarbonQueryUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/CarbonQueryUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/CarbonQueryUtil.java
index 8db32a2..823aad3 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/CarbonQueryUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/CarbonQueryUtil.java
@@ -26,8 +26,8 @@ import java.util.List;
import java.util.Map;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.updatestatus.LoadMetadataDetails;
import org.apache.carbondata.core.scan.model.CarbonQueryPlan;
+import org.apache.carbondata.core.updatestatus.LoadMetadataDetails;
import org.apache.carbondata.spark.partition.api.Partition;
import org.apache.carbondata.spark.partition.api.impl.DefaultLoadBalancer;
import org.apache.carbondata.spark.partition.api.impl.PartitionMultiFileImpl;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/LoadMetadataUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/LoadMetadataUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/LoadMetadataUtil.java
index e8d4dde..0055f52 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/LoadMetadataUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/util/LoadMetadataUtil.java
@@ -28,9 +28,9 @@
*/
package org.apache.carbondata.spark.util;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.updatestatus.LoadMetadataDetails;
import org.apache.carbondata.core.updatestatus.SegmentStatusManager;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
index df3ee75..4ab7ffd 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
@@ -27,8 +27,8 @@ import org.apache.spark.sql.catalyst.expressions.{Cast, Literal}
import org.apache.spark.sql.types.TimestampType
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata}
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata}
import org.apache.carbondata.core.updatestatus.SegmentStatusManager
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.rdd.DataManagementFunc
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonSparkFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonSparkFactory.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonSparkFactory.scala
index 89cbe41..735895a 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonSparkFactory.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonSparkFactory.scala
@@ -16,8 +16,8 @@
*/
package org.apache.carbondata.spark
-import org.apache.carbondata.core.metadata.schema.table.column.{CarbonDimension, ColumnSchema}
import org.apache.carbondata.core.metadata.{CarbonTableIdentifier, ColumnIdentifier}
+import org.apache.carbondata.core.metadata.schema.table.column.{CarbonDimension, ColumnSchema}
/**
* Column validator
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
index 9ebda94..e0dbb54 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
@@ -19,10 +19,10 @@ package org.apache.carbondata.spark
import scala.collection.mutable.HashMap
-import org.apache.carbondata.core.datastorage.filesystem.{CarbonFile, CarbonFileFilter}
-import org.apache.carbondata.core.datastorage.impl.FileFactory
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
+import org.apache.carbondata.core.datastore.filesystem.{CarbonFile, CarbonFileFilter}
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.{CarbonTableIdentifier, ColumnIdentifier}
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.path.{CarbonStorePath, CarbonTablePath}
class DictionaryDetailHelper extends DictionaryDetailService {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala
index 796ef7d..5ebb124 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala
@@ -46,8 +46,7 @@ import org.apache.carbondata.processing.model.CarbonLoadModel
import org.apache.carbondata.spark.DataLoadResult
import org.apache.carbondata.spark.load._
import org.apache.carbondata.spark.splits.TableSplit
-import org.apache.carbondata.spark.util.CarbonQueryUtil
-import org.apache.carbondata.spark.util.CarbonScalaUtil
+import org.apache.carbondata.spark.util.{CarbonQueryUtil, CarbonScalaUtil}
/**
* This partition class use to split by TableSplit
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
index c6247d0..77499a6 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
@@ -33,16 +33,15 @@ import org.apache.spark.rdd.RDD
import org.apache.spark.sql.Row
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.metadata.ColumnIdentifier
import org.apache.carbondata.core.cache.dictionary.Dictionary
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
-import org.apache.carbondata.core.path.CarbonTablePath
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastorage.impl.FileFactory
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.{CarbonTableIdentifier, ColumnIdentifier}
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
+import org.apache.carbondata.core.path.CarbonTablePath
import org.apache.carbondata.core.service.{CarbonCommonFactory, PathService}
-import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory, CarbonUtil}
import org.apache.carbondata.core.updatestatus.locks.{CarbonLockFactory, LockUsage}
+import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory, CarbonUtil}
import org.apache.carbondata.processing.model.CarbonLoadModel
import org.apache.carbondata.spark.load.CarbonLoaderUtil
import org.apache.carbondata.spark.tasks.{DictionaryWriterTask, SortIndexWriterTask}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
index 11eb0c8..d4918ce 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
@@ -27,7 +27,6 @@ import org.apache.hadoop.mapreduce.Job
import org.apache.spark.{Partition, SparkContext}
import org.apache.spark.sql.execution.command.CarbonMergerMapping
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier
import org.apache.carbondata.core.datastore.block.{Distributable, TableBlockInfo}
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.hadoop.{CarbonInputFormat, CarbonInputSplit, CarbonMultiBlockSplit}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index 523ed96..b1e4547 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -36,15 +36,15 @@ import org.apache.spark.sql.hive.DistributionUtil
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.block._
-import org.apache.carbondata.core.metadata.blocklet.DataFileFooter
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
+import org.apache.carbondata.core.metadata.blocklet.DataFileFooter
import org.apache.carbondata.core.path.CarbonTablePath
import org.apache.carbondata.core.scan.result.iterator.RawResultIterator
import org.apache.carbondata.core.update.data.UpdateVO
import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
-import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil
import org.apache.carbondata.hadoop.{CarbonInputFormat, CarbonInputSplit, CarbonMultiBlockSplit}
+import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil
import org.apache.carbondata.processing.model.CarbonLoadModel
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil
import org.apache.carbondata.spark.MergeResult
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
index c7bbfa9..b06fe14 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
@@ -34,11 +34,11 @@ import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.datastore.block.Distributable
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.core.scan.expression.Expression
+import org.apache.carbondata.core.scan.model.QueryModel
import org.apache.carbondata.core.stats.{QueryStatistic, QueryStatisticsConstants, QueryStatisticsRecorder}
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory
import org.apache.carbondata.hadoop._
-import org.apache.carbondata.core.scan.expression.Expression
-import org.apache.carbondata.core.scan.model.QueryModel
import org.apache.carbondata.spark.load.CarbonLoaderUtil
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/Compactor.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/Compactor.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/Compactor.scala
index a2941e5..124b4b2 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/Compactor.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/Compactor.scala
@@ -25,7 +25,6 @@ import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.update.CarbonUpdateUtil
import org.apache.carbondata.core.updatestatus.SegmentStatusManager
import org.apache.carbondata.spark.MergeResultImpl
-import org.apache.carbondata.spark.load.CarbonLoaderUtil
import org.apache.carbondata.spark.merger.{CarbonDataMergerUtil, CompactionType}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
index e4d0671..3e4c5ac 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
@@ -27,9 +27,9 @@ import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.execution.command.{CompactionCallableModel, CompactionModel}
import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.metadata.{CarbonMetadata, CarbonTableIdentifier}
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.update.CarbonUpdateUtil
import org.apache.carbondata.core.updatestatus.{LoadMetadataDetails, SegmentStatusManager}
import org.apache.carbondata.core.updatestatus.locks.{CarbonLockFactory, CarbonLockUtil, LockUsage}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
index e63f904..645294f 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
@@ -31,9 +31,10 @@ import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.mapreduce.{TaskAttemptID, TaskType}
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
import org.apache.spark.{Partition, SparkContext, SparkEnv, TaskContext}
-import org.apache.spark.rdd.{DataLoadCoalescedRDD, DataLoadPartitionWrap, RDD}
+import org.apache.spark.rdd.DataLoadCoalescedRDD
+import org.apache.spark.rdd.DataLoadPartitionWrap
+import org.apache.spark.rdd.RDD
import org.apache.spark.sql.Row
-import org.apache.spark.sql.execution.command.Partitioner
import org.apache.spark.util.SparkUtil
import org.apache.carbondata.common.CarbonIterator
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/UpdateCoalescedRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/UpdateCoalescedRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/UpdateCoalescedRDD.scala
index 67e094a..378d013 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/UpdateCoalescedRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/UpdateCoalescedRDD.scala
@@ -20,7 +20,6 @@ package org.apache.spark.rdd
import scala.reflect.ClassTag
import org.apache.spark._
-import org.apache.spark.rdd.{CoalescedRDDPartition, DataLoadPartitionCoalescer, RDD}
// This RDD distributes previous RDD data based on number of nodes. i.e., one partition for one node
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
index 6712f7e..c85c466 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
@@ -24,9 +24,9 @@ import org.apache.spark.sql._
import org.apache.spark.sql.types._
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.metadata.{DataType => CarbonDataType}
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastorage.impl.FileFactory
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.metadata.{DataType => CarbonDataType}
import org.apache.carbondata.core.util.CarbonProperties
object CarbonScalaUtil {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
index 708e9b6..62a7982 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -17,7 +17,7 @@
package org.apache.carbondata.spark.util
-import java.io.{FileNotFoundException, IOException}
+import java.io.FileNotFoundException
import java.nio.charset.Charset
import java.util.regex.Pattern
@@ -38,20 +38,19 @@ import org.apache.spark.util.FileUtils
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.cache.dictionary.Dictionary
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.{CarbonTableIdentifier, DataType, Encoding}
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.path.CarbonStorePath
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastorage.impl.FileFactory
import org.apache.carbondata.core.reader.CarbonDictionaryReader
import org.apache.carbondata.core.service.CarbonCommonFactory
-import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
+import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.writer.CarbonDictionaryWriter
import org.apache.carbondata.hadoop.csv.CSVInputFormat
import org.apache.carbondata.hadoop.io.StringArrayWritable
import org.apache.carbondata.processing.etl.DataLoadingException
import org.apache.carbondata.processing.model.CarbonLoadModel
-import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException
import org.apache.carbondata.spark.CarbonSparkFactory
import org.apache.carbondata.spark.load.CarbonLoaderUtil
import org.apache.carbondata.spark.rdd._
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 5114e29..c389ae3 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -31,8 +31,8 @@ import org.apache.spark.sql.catalyst.trees.CurrentOrigin
import org.apache.spark.sql.execution.command._
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.metadata.DataType
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.DataType
import org.apache.carbondata.core.util.DataTypeUtil
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.util.CommonUtil
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index ba468f8..fc1d56d 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -26,12 +26,11 @@ import scala.collection.mutable.Map
import org.apache.spark.sql.SQLContext
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.metadata.DataType
-import org.apache.carbondata.core.metadata.Encoding
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.{DataType, Encoding}
import org.apache.carbondata.core.metadata.schema.{BucketingInfo, SchemaEvolution, SchemaEvolutionEntry}
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo, TableSchema}
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
-import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.service.CarbonCommonFactory
import org.apache.carbondata.core.updatestatus.{LoadMetadataDetails, SegmentUpdateStatusManager}
import org.apache.carbondata.processing.model.CarbonLoadModel
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common/src/main/scala/org/apache/spark/sql/hive/DistributionUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/hive/DistributionUtil.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/hive/DistributionUtil.scala
index 581338e..afcd970 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/hive/DistributionUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/hive/DistributionUtil.scala
@@ -25,8 +25,8 @@ import org.apache.spark.SparkContext
import org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.datastore.block.Distributable
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.block.Distributable
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.spark.load.CarbonLoaderUtil
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala b/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala
index 7a19a92..3060ff8 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/util/FileUtils.scala
@@ -19,8 +19,8 @@ package org.apache.spark.util
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFile
-import org.apache.carbondata.core.datastorage.impl.FileFactory
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.processing.etl.DataLoadingException
object FileUtils {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
index 095ac26..4387ccc 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/CarbonDataFrameWriter.scala
@@ -24,8 +24,8 @@ import org.apache.spark.sql.execution.command.LoadTable
import org.apache.spark.sql.types._
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.metadata.{DataType => CarbonType}
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.{DataType => CarbonType}
class CarbonDataFrameWriter(val dataFrame: DataFrame) {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 142796a..8fe690b 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -30,19 +30,19 @@ import org.apache.hadoop.conf.{Configurable, Configuration}
import org.apache.hadoop.fs.Path
import org.apache.hadoop.mapreduce.Job
import org.apache.hadoop.mapreduce.lib.input.{FileInputFormat, FileSplit}
+import org.apache.spark.{SparkEnv, SparkException}
import org.apache.spark.rdd.{DataLoadCoalescedRDD, DataLoadPartitionCoalescer, UpdateCoalescedRDD}
+import org.apache.spark.sql.{CarbonEnv, DataFrame, Row, SQLContext}
import org.apache.spark.sql.execution.command.{AlterTableModel, CompactionModel, ExecutionErrors, UpdateTableModel}
import org.apache.spark.sql.hive.DistributionUtil
-import org.apache.spark.sql.{CarbonEnv, DataFrame, Row, SQLContext}
import org.apache.spark.util.SparkUtil
-import org.apache.spark.{SparkEnv, SparkException}
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.block.{Distributable, TableBlockInfo}
import org.apache.carbondata.core.dictionary.server.DictionaryServer
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.metadata.{CarbonTableIdentifier, ColumnarFormatVersion}
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.path.CarbonStorePath
import org.apache.carbondata.core.update.CarbonUpdateUtil
import org.apache.carbondata.core.updatestatus.LoadMetadataDetails
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
index eceb3a4..12ab6cb 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
@@ -37,9 +37,9 @@ import org.apache.spark.util.SerializableConfiguration
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.path.CarbonTablePath
+import org.apache.carbondata.core.scan.expression.logical.AndExpression
import org.apache.carbondata.hadoop.{CarbonInputFormat, CarbonInputSplit, CarbonProjection}
import org.apache.carbondata.hadoop.util.{CarbonInputFormatUtil, SchemaReader}
-import org.apache.carbondata.core.scan.expression.logical.AndExpression
import org.apache.carbondata.spark.{CarbonFilters, CarbonOption}
import org.apache.carbondata.spark.merger.TableMeta
import org.apache.carbondata.spark.readsupport.SparkRowReadSupportImpl
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
index 5bcaccd..6559473 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
@@ -29,7 +29,7 @@ import org.apache.spark.sql.hive.{CarbonMetaData, CarbonMetastoreTypes}
import org.apache.spark.sql.sources._
import org.apache.spark.sql.types.{DataType, StructType}
-import org.apache.carbondata.core.datastorage.impl.FileFactory
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, CarbonDimension}
import org.apache.carbondata.core.metadata.schema.table.column.CarbonImplicitDimension
import org.apache.carbondata.core.path.CarbonStorePath
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
index 476c74c..b878326 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
@@ -29,8 +29,8 @@ import org.apache.spark.sql.types._
import org.apache.carbondata.core.cache.{Cache, CacheProvider, CacheType}
import org.apache.carbondata.core.cache.dictionary.{Dictionary, DictionaryColumnUniqueIdentifier}
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, ColumnIdentifier, DataType, Encoding}
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.stats.{QueryStatistic, QueryStatisticsConstants}
import org.apache.carbondata.core.util.{CarbonTimeStatisticsFactory, DataTypeUtil}
import org.apache.carbondata.spark.CarbonAliasDecoderRelation
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
index d080bde..5006508 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonScan.scala
@@ -26,8 +26,8 @@ import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.execution.LeafNode
import org.apache.spark.sql.hive.CarbonMetastore
-import org.apache.carbondata.hadoop.CarbonProjection
import org.apache.carbondata.core.scan.model._
+import org.apache.carbondata.hadoop.CarbonProjection
import org.apache.carbondata.spark.CarbonFilters
import org.apache.carbondata.spark.rdd.CarbonScanRDD
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
index a0c3831..156153d 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
@@ -17,8 +17,6 @@
package org.apache.spark.sql.execution.command
-import java.io.IOException
-import java.text.SimpleDateFormat
import java.util
import scala.collection.JavaConverters._
@@ -31,25 +29,24 @@ import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project}
import org.apache.spark.sql.execution.RunnableCommand
import org.apache.spark.sql.execution.datasources.LogicalRelation
import org.apache.spark.storage.StorageLevel
-import org.apache.spark.util.FileUtils
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.path.{CarbonStorePath, CarbonTablePath}
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastorage.impl.FileFactory
+import org.apache.carbondata.core.datastore.impl.FileFactory
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.core.path.{CarbonStorePath, CarbonTablePath}
import org.apache.carbondata.core.update._
import org.apache.carbondata.core.update.data.RowCountDetailsVO
import org.apache.carbondata.core.updatestatus.{SegmentStatusManager, SegmentUpdateStatusManager}
+import org.apache.carbondata.core.updatestatus.locks.{CarbonLockFactory, CarbonLockUtil, LockUsage}
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.writer.CarbonDeleteDeltaWriterImpl
-import org.apache.carbondata.core.updatestatus.locks.{CarbonLockFactory, CarbonLockUtil, LockUsage}
import org.apache.carbondata.processing.exception.MultipleMatchingException
-import org.apache.carbondata.spark.load.{CarbonLoaderUtil, FailureCauses}
+import org.apache.carbondata.spark.DeleteDelataResultImpl
+import org.apache.carbondata.spark.load.FailureCauses
import org.apache.carbondata.spark.merger.{CarbonDataMergerUtil, CarbonDataMergerUtilResult, CompactionType}
import org.apache.carbondata.spark.merger.CarbonDataMergerUtil._
-import org.apache.carbondata.spark.DeleteDelataResultImpl
import org.apache.carbondata.spark.util.QueryPlanUtil
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 93fb660..323c090 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -30,7 +30,7 @@ import org.apache.commons.lang3.StringUtils
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
-import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Cast, Literal}
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.{RunnableCommand, SparkPlan}
import org.apache.spark.sql.hive.CarbonMetastore
@@ -40,17 +40,16 @@ import org.codehaus.jackson.map.ObjectMapper
import org.apache.carbondata.api.CarbonStore
import org.apache.carbondata.common.logging.LogServiceFactory
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.dictionary.server.DictionaryServer
import org.apache.carbondata.core.metadata.{CarbonTableIdentifier, Encoding}
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.path.CarbonStorePath
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastorage.impl.FileFactory
-import org.apache.carbondata.core.dictionary.server.DictionaryServer
-import org.apache.carbondata.core.update.CarbonUpdateUtil
-import org.apache.carbondata.core.update.TupleIdEnum
-import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
+import org.apache.carbondata.core.update.{CarbonUpdateUtil, TupleIdEnum}
import org.apache.carbondata.core.updatestatus.locks.{CarbonLockFactory, LockUsage}
+import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
import org.apache.carbondata.processing.constants.TableOptionConstant
import org.apache.carbondata.processing.etl.DataLoadingException
import org.apache.carbondata.processing.model.{CarbonDataLoadSchema, CarbonLoadModel}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
index 123bef9..970a7ea 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
@@ -35,18 +35,18 @@ import org.apache.spark.sql.types._
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastorage.impl.FileFactory
-import org.apache.carbondata.core.datastorage.impl.FileFactory.FileType
+import org.apache.carbondata.core.datastore.impl.FileFactory
+import org.apache.carbondata.core.datastore.impl.FileFactory.FileType
import org.apache.carbondata.core.metadata.{CarbonMetadata, CarbonTableIdentifier}
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.path.{CarbonStorePath, CarbonTablePath}
import org.apache.carbondata.core.reader.ThriftReader
import org.apache.carbondata.core.stats.{QueryStatistic, QueryStatisticsConstants}
+import org.apache.carbondata.core.updatestatus.locks.ZookeeperInit
import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory, CarbonUtil}
import org.apache.carbondata.core.writer.ThriftWriter
import org.apache.carbondata.format.{SchemaEvolutionEntry, TableInfo}
-import org.apache.carbondata.core.updatestatus.locks.ZookeeperInit
import org.apache.carbondata.spark.merger.TableMeta
case class MetaData(var tablesMeta: ArrayBuffer[TableMeta])
[50/57] [abbrv] incubator-carbondata git commit: fix style
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
deleted file mode 100644
index e47cf42..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.impl.data.compressed;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.NodeMeasureDataStore;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-
-public abstract class AbstractHeavyCompressedDoubleArrayDataStore
- implements NodeMeasureDataStore //NodeMeasureDataStore<double[]>
-{
-
- private LogService LOGGER =
- LogServiceFactory.getLogService(AbstractHeavyCompressedDoubleArrayDataStore.class.getName());
-
- /**
- * values.
- */
- protected ValueCompressionHolder[] values;
-
- /**
- * compressionModel.
- */
- protected WriterCompressModel compressionModel;
-
- /**
- * type
- */
- private char[] type;
-
- /**
- * AbstractHeavyCompressedDoubleArrayDataStore constructor.
- *
- * @param compressionModel
- */
- public AbstractHeavyCompressedDoubleArrayDataStore(WriterCompressModel compressionModel) {
- this.compressionModel = compressionModel;
- if (null != compressionModel) {
- this.type = compressionModel.getType();
- values =
- new ValueCompressionHolder[compressionModel.getValueCompressionHolder().length];
- }
- }
-
- // this method first invokes encoding routine to encode the data chunk,
- // followed by invoking compression routine for preparing the data chunk for writing.
- @Override public byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] dataHolder) {
- byte[][] returnValue = new byte[values.length][];
- for (int i = 0; i < compressionModel.getValueCompressionHolder().length; i++) {
- values[i] = compressionModel.getValueCompressionHolder()[i];
- if (type[i] != CarbonCommonConstants.BYTE_VALUE_MEASURE) {
- // first perform encoding of the data chunk
- values[i].setValue(
- ValueCompressionUtil.getValueCompressor(compressionModel.getCompressionFinders()[i])
- .getCompressedValues(compressionModel.getCompressionFinders()[i], dataHolder[i],
- compressionModel.getMaxValue()[i],
- compressionModel.getMantissa()[i]));
- } else {
- values[i].setValue(dataHolder[i].getWritableByteArrayValues());
- }
- values[i].compress();
- returnValue[i] = values[i].getCompressedData();
- }
-
- return returnValue;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
deleted file mode 100644
index ea42beb..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.impl.data.compressed;
-
-import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
-
-public class HeavyCompressedDoubleArrayDataInMemoryStore
- extends AbstractHeavyCompressedDoubleArrayDataStore {
-
- public HeavyCompressedDoubleArrayDataInMemoryStore(WriterCompressModel compressionModel) {
- super(compressionModel);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
index 9d4287a..7acfb39 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
@@ -20,7 +20,6 @@ package org.apache.carbondata.core.datastore;
import java.io.IOException;
-import org.apache.carbondata.core.datastorage.FileHolder;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
import org.apache.carbondata.core.update.data.BlockletLevelDeleteDeltaDataCache;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java b/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java
new file mode 100644
index 0000000..95ec433
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/FileHolder.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore;
+
+import java.io.IOException;
+
+public interface FileHolder {
+ /**
+ * This method will be used to read the byte array from file based on offset
+ * and length(number of bytes) need to read
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @param length number of bytes to be read
+ * @return read byte array
+ */
+ byte[] readByteArray(String filePath, long offset, int length) throws IOException;
+
+ /**
+ * This method will be used to read the byte array from file based on length(number of bytes)
+ *
+ * @param filePath fully qualified file path
+ * @param length number of bytes to be read
+ * @return read byte array
+ */
+ byte[] readByteArray(String filePath, int length) throws IOException;
+
+ /**
+ * This method will be used to read int from file from postion(offset), here
+ * length will be always 4 bacause int byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read int
+ */
+ int readInt(String filePath, long offset) throws IOException;
+
+ /**
+ * This method will be used to read long from file from postion(offset), here
+ * length will be always 8 bacause int byte size is 8
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read long
+ */
+ long readLong(String filePath, long offset) throws IOException;
+
+ /**
+ * This method will be used to read int from file from postion(offset), here
+ * length will be always 4 bacause int byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @return read int
+ */
+ int readInt(String filePath) throws IOException;
+
+ /**
+ * This method will be used to read long value from file from postion(offset), here
+ * length will be always 8 because long byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read long
+ */
+ long readDouble(String filePath, long offset) throws IOException;
+
+ /**
+ * This method will be used to close all the streams currently present in the cache
+ */
+ void finish() throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/MeasureDataWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/MeasureDataWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastore/MeasureDataWrapper.java
new file mode 100644
index 0000000..ff6ed34
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/MeasureDataWrapper.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore;
+
+import org.apache.carbondata.core.datastore.dataholder.CarbonReadDataHolder;
+
+/**
+ * MeasureDataWrapper, interface.
+ */
+public interface MeasureDataWrapper {
+ CarbonReadDataHolder[] getValues();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/NodeMeasureDataStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/NodeMeasureDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/NodeMeasureDataStore.java
new file mode 100644
index 0000000..c4e5d15
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/NodeMeasureDataStore.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore;
+
+import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder;
+
+public interface NodeMeasureDataStore {
+ /**
+ * This method will be used to get the writable key array.
+ * writable measure data array will hold below information:
+ * <size of measure data array><measure data array>
+ * total length will be 4 bytes for size + measure data array length
+ *
+ * @return writable array (compressed or normal)
+ */
+ byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] dataHolderArray);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
index dde8e76..f8c4374 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
@@ -29,11 +29,10 @@ import java.util.Map.Entry;
import java.util.Set;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.columnar.ColumnGroupModel;
+import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.keygenerator.columnar.ColumnarSplitter;
-import org.apache.carbondata.core.keygenerator.columnar.impl
- .MultiDimKeyVarLengthVariableSplitGenerator;
+import org.apache.carbondata.core.keygenerator.columnar.impl.MultiDimKeyVarLengthVariableSplitGenerator;
import org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.metadata.Encoding;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
index c7ce3db..8774579 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
@@ -23,7 +23,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.path.CarbonTablePath.DataFileUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java
index ce09305..3c08f45 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java
@@ -18,7 +18,7 @@
*/
package org.apache.carbondata.core.datastore.chunk;
-import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.datastore.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.metadata.blocklet.datachunk.PresenceMeta;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
index 6edc977..0730607 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
@@ -20,8 +20,7 @@ package org.apache.carbondata.core.datastore.chunk.impl;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory
- .DimensionStoreType;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
index 549d6d8..342c392 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
@@ -22,8 +22,7 @@ import java.util.Arrays;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory
- .DimensionStoreType;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
index c83f2d7..3d4e3bf 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
@@ -18,14 +18,10 @@
*/
package org.apache.carbondata.core.datastore.chunk.reader;
-import org.apache.carbondata.core.datastore.chunk.reader.dimension
- .v1.CompressedDimensionChunkFileBasedReaderV1;
-import org.apache.carbondata.core.datastore.chunk.reader.dimension
- .v2.CompressedDimensionChunkFileBasedReaderV2;
-import org.apache.carbondata.core.datastore.chunk.reader.measure
- .v1.CompressedMeasureChunkFileBasedReaderV1;
-import org.apache.carbondata.core.datastore.chunk.reader.measure
- .v2.CompressedMeasureChunkFileBasedReaderV2;
+import org.apache.carbondata.core.datastore.chunk.reader.dimension.v1.CompressedDimensionChunkFileBasedReaderV1;
+import org.apache.carbondata.core.datastore.chunk.reader.dimension.v2.CompressedDimensionChunkFileBasedReaderV2;
+import org.apache.carbondata.core.datastore.chunk.reader.measure.v1.CompressedMeasureChunkFileBasedReaderV1;
+import org.apache.carbondata.core.datastore.chunk.reader.measure.v2.CompressedMeasureChunkFileBasedReaderV2;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
index 92dcb60..4981241 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
@@ -20,7 +20,7 @@ package org.apache.carbondata.core.datastore.chunk.reader;
import java.io.IOException;
-import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
index 94004d9..a706d71 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
@@ -20,7 +20,7 @@ package org.apache.carbondata.core.datastore.chunk.reader;
import java.io.IOException;
-import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java
index 4cc24d9..2de673a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java
@@ -19,9 +19,9 @@
package org.apache.carbondata.core.datastore.chunk.reader.dimension;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
import org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader;
+import org.apache.carbondata.core.datastore.compression.Compressor;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
import org.apache.carbondata.core.util.CarbonProperties;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
index dd487d9..a8d15fe 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
@@ -21,13 +21,13 @@ package org.apache.carbondata.core.datastore.chunk.reader.dimension.v1;
import java.io.IOException;
import java.util.List;
-import org.apache.carbondata.core.datastorage.FileHolder;
-import org.apache.carbondata.core.datastorage.columnar.UnBlockIndexer;
+import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.ColumnGroupDimensionDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
import org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReader;
+import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
index 3bddf4e..ae5c9b4 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
@@ -21,13 +21,13 @@ package org.apache.carbondata.core.datastore.chunk.reader.dimension.v2;
import java.io.IOException;
import java.util.List;
-import org.apache.carbondata.core.datastorage.FileHolder;
-import org.apache.carbondata.core.datastorage.columnar.UnBlockIndexer;
+import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.ColumnGroupDimensionDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
import org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReader;
+import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.format.DataChunk2;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
index 48cb244..1f3fec0 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
@@ -21,12 +21,12 @@ package org.apache.carbondata.core.datastore.chunk.reader.measure.v1;
import java.io.IOException;
import java.util.List;
-import org.apache.carbondata.core.datastorage.FileHolder;
-import org.apache.carbondata.core.datastorage.compression.ReaderCompressModel;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReader;
+import org.apache.carbondata.core.datastore.compression.ReaderCompressModel;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
index 3325299..2731163 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
@@ -23,13 +23,13 @@ import java.util.ArrayList;
import java.util.BitSet;
import java.util.List;
-import org.apache.carbondata.core.datastorage.FileHolder;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReader;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastore.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.metadata.blocklet.datachunk.PresenceMeta;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java
index df054c5..e1ae9b3 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java
@@ -17,14 +17,10 @@
package org.apache.carbondata.core.datastore.chunk.store;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.chunk.store.impl.safe
- .SafeFixedLengthDimensionDataChunkStore;
-import org.apache.carbondata.core.datastore.chunk.store.impl.safe
- .SafeVariableLengthDimensionDataChunkStore;
-import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe
- .UnsafeFixedLengthDimensionDataChunkStore;
-import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe
- .UnsafeVariableLengthDimesionDataChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeFixedLengthDimensionDataChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeVariableLengthDimensionDataChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeFixedLengthDimensionDataChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeVariableLengthDimesionDataChunkStore;
import org.apache.carbondata.core.util.CarbonProperties;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java
new file mode 100644
index 0000000..b6155fb
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForInt.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.ByteUtil;
+
+public class BlockIndexerStorageForInt implements IndexStorage<int[]> {
+ private boolean alreadySorted;
+
+ private int[] dataAfterComp;
+
+ private int[] indexMap;
+
+ private byte[][] keyBlock;
+
+ private int[] dataIndexMap;
+
+ private int totalSize;
+
+ public BlockIndexerStorageForInt(byte[][] keyBlock, boolean compressData, boolean isNoDictionary,
+ boolean isSortRequired) {
+ ColumnWithIntIndex[] columnWithIndexs = createColumnWithIndexArray(keyBlock, isNoDictionary);
+ if (isSortRequired) {
+ Arrays.sort(columnWithIndexs);
+ }
+ compressMyOwnWay(extractDataAndReturnIndexes(columnWithIndexs, keyBlock));
+ if (compressData) {
+ compressDataMyOwnWay(columnWithIndexs);
+ }
+ }
+
+ /**
+ * Create an object with each column array and respective index
+ *
+ * @return
+ */
+ private ColumnWithIntIndex[] createColumnWithIndexArray(byte[][] keyBlock,
+ boolean isNoDictionary) {
+ ColumnWithIntIndex[] columnWithIndexs;
+ if (isNoDictionary) {
+ columnWithIndexs = new ColumnWithIntIndexForHighCard[keyBlock.length];
+ for (int i = 0; i < columnWithIndexs.length; i++) {
+ columnWithIndexs[i] = new ColumnWithIntIndexForHighCard(keyBlock[i], i);
+ }
+
+ } else {
+ columnWithIndexs = new ColumnWithIntIndex[keyBlock.length];
+ for (int i = 0; i < columnWithIndexs.length; i++) {
+ columnWithIndexs[i] = new ColumnWithIntIndex(keyBlock[i], i);
+ }
+ }
+
+ return columnWithIndexs;
+ }
+
+ private int[] extractDataAndReturnIndexes(ColumnWithIntIndex[] columnWithIndexs,
+ byte[][] keyBlock) {
+ int[] indexes = new int[columnWithIndexs.length];
+ for (int i = 0; i < indexes.length; i++) {
+ indexes[i] = columnWithIndexs[i].getIndex();
+ keyBlock[i] = columnWithIndexs[i].getColumn();
+ }
+ this.keyBlock = keyBlock;
+ return indexes;
+ }
+
+ /**
+ * It compresses depends up on the sequence numbers.
+ * [1,2,3,4,6,8,10,11,12,13] is translated to [1,4,6,8,10,13] and [0,6]. In
+ * first array the start and end of sequential numbers and second array
+ * keeps the indexes of where sequential numbers starts. If there is no
+ * sequential numbers then the same array it returns with empty second
+ * array.
+ *
+ * @param indexes
+ */
+ public void compressMyOwnWay(int[] indexes) {
+ List<Integer> list = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ int k = 0;
+ int i = 1;
+ for (; i < indexes.length; i++) {
+ if (indexes[i] - indexes[i - 1] == 1) {
+ k++;
+ } else {
+ if (k > 0) {
+ map.add((list.size()));
+ list.add(indexes[i - k - 1]);
+ list.add(indexes[i - 1]);
+ } else {
+ list.add(indexes[i - 1]);
+ }
+ k = 0;
+ }
+ }
+ if (k > 0) {
+ map.add((list.size()));
+ list.add(indexes[i - k - 1]);
+ list.add(indexes[i - 1]);
+ } else {
+ list.add(indexes[i - 1]);
+ }
+ dataAfterComp = convertToArray(list);
+ if (indexes.length == dataAfterComp.length) {
+ indexMap = new int[0];
+ } else {
+ indexMap = convertToArray(map);
+ }
+ if (dataAfterComp.length == 2 && indexMap.length == 1) {
+ alreadySorted = true;
+ }
+ }
+
+ private int[] convertToArray(List<Integer> list) {
+ int[] shortArray = new int[list.size()];
+ for (int i = 0; i < shortArray.length; i++) {
+ shortArray[i] = list.get(i);
+ }
+ return shortArray;
+ }
+
+ /**
+ * @return the alreadySorted
+ */
+ public boolean isAlreadySorted() {
+ return alreadySorted;
+ }
+
+ /**
+ * @return the dataAfterComp
+ */
+ public int[] getDataAfterComp() {
+ return dataAfterComp;
+ }
+
+ /**
+ * @return the indexMap
+ */
+ public int[] getIndexMap() {
+ return indexMap;
+ }
+
+ /**
+ * @return the keyBlock
+ */
+ public byte[][] getKeyBlock() {
+ return keyBlock;
+ }
+
+ private void compressDataMyOwnWay(ColumnWithIntIndex[] indexes) {
+ byte[] prvKey = indexes[0].getColumn();
+ List<ColumnWithIntIndex> list =
+ new ArrayList<ColumnWithIntIndex>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ list.add(indexes[0]);
+ int counter = 1;
+ int start = 0;
+ List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ for (int i = 1; i < indexes.length; i++) {
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, indexes[i].getColumn()) != 0) {
+ prvKey = indexes[i].getColumn();
+ list.add(indexes[i]);
+ map.add(start);
+ map.add(counter);
+ start += counter;
+ counter = 1;
+ continue;
+ }
+ counter++;
+ }
+ map.add(start);
+ map.add(counter);
+ this.keyBlock = convertToKeyArray(list);
+ if (indexes.length == keyBlock.length) {
+ dataIndexMap = new int[0];
+ } else {
+ dataIndexMap = convertToArray(map);
+ }
+ }
+
+ private byte[][] convertToKeyArray(List<ColumnWithIntIndex> list) {
+ byte[][] shortArray = new byte[list.size()][];
+ for (int i = 0; i < shortArray.length; i++) {
+ shortArray[i] = list.get(i).getColumn();
+ totalSize += shortArray[i].length;
+ }
+ return shortArray;
+ }
+
+ @Override public int[] getDataIndexMap() {
+ return dataIndexMap;
+ }
+
+ @Override public int getTotalSize() {
+ return totalSize;
+ }
+
+ @Override public byte[] getMin() {
+ return keyBlock[0];
+ }
+
+ @Override public byte[] getMax() {
+ return keyBlock[keyBlock.length - 1];
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
new file mode 100644
index 0000000..d7b980e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForNoInvertedIndex.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.ByteUtil;
+
+public class BlockIndexerStorageForNoInvertedIndex implements IndexStorage<int[]> {
+ private byte[][] keyBlock;
+ private byte[][] sortedBlock;
+ private int totalSize;
+ private int[] dataIndexMap;
+
+ public BlockIndexerStorageForNoInvertedIndex(byte[][] keyBlockInput, boolean compressData,
+ boolean isNoDictionary) {
+ // without invertedindex but can be RLE
+ if (compressData) {
+ // with RLE
+ byte[] prvKey = keyBlockInput[0];
+ List<byte[]> list = new ArrayList<byte[]>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ list.add(keyBlockInput[0]);
+ int counter = 1;
+ int start = 0;
+ List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ int length = keyBlockInput.length;
+ for(int i = 1; i < length; i++) {
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, keyBlockInput[i]) != 0) {
+ prvKey = keyBlockInput[i];
+ list.add(keyBlockInput[i]);
+ map.add(start);
+ map.add(counter);
+ start += counter;
+ counter = 1;
+ continue;
+ }
+ counter++;
+ }
+ map.add(start);
+ map.add(counter);
+ this.keyBlock = convertToKeyArray(list);
+ if (keyBlockInput.length == this.keyBlock.length) {
+ dataIndexMap = new int[0];
+ } else {
+ dataIndexMap = convertToArray(map);
+ }
+ } else {
+ this.keyBlock = keyBlockInput;
+ dataIndexMap = new int[0];
+ }
+
+ this.sortedBlock = new byte[keyBlock.length][];
+ System.arraycopy(keyBlock, 0, sortedBlock, 0, keyBlock.length);
+ if (isNoDictionary) {
+ Arrays.sort(sortedBlock, new Comparator<byte[]>() {
+ @Override
+ public int compare(byte[] col1, byte[] col2) {
+ return ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(col1, 2, col1.length - 2, col2, 2, col2.length - 2);
+ }
+ });
+ } else {
+ Arrays.sort(sortedBlock, new Comparator<byte[]>() {
+ @Override
+ public int compare(byte[] col1, byte[] col2) {
+ return ByteUtil.UnsafeComparer.INSTANCE.compareTo(col1, col2);
+ }
+ });
+ }
+
+ }
+
+ private int[] convertToArray(List<Integer> list) {
+ int[] shortArray = new int[list.size()];
+ for(int i = 0; i < shortArray.length; i++) {
+ shortArray[i] = list.get(i);
+ }
+ return shortArray;
+ }
+
+ private byte[][] convertToKeyArray(List<byte[]> list) {
+ byte[][] shortArray = new byte[list.size()][];
+ for (int i = 0; i < shortArray.length; i++) {
+ shortArray[i] = list.get(i);
+ totalSize += shortArray[i].length;
+ }
+ return shortArray;
+ }
+
+ @Override
+ public int[] getDataIndexMap() {
+ return dataIndexMap;
+ }
+
+ @Override
+ public int getTotalSize() {
+ return totalSize;
+ }
+
+ @Override
+ public boolean isAlreadySorted() {
+ return true;
+ }
+
+ /**
+ * no use
+ * @return
+ */
+ @Override
+ public int[] getDataAfterComp() {
+ return new int[0];
+ }
+
+ /**
+ * no use
+ * @return
+ */
+ @Override
+ public int[] getIndexMap() {
+ return new int[0];
+ }
+
+ /**
+ * @return the keyBlock
+ */
+ public byte[][] getKeyBlock() {
+ return keyBlock;
+ }
+
+ @Override public byte[] getMin() {
+ return sortedBlock[0];
+ }
+
+ @Override public byte[] getMax() {
+ return sortedBlock[sortedBlock.length - 1];
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
new file mode 100644
index 0000000..4d4c19d
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnGroupModel.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.datastore.columnar;
+
+public class ColumnGroupModel {
+
+ /**
+ * number of columns in columnar block
+ */
+ private int[] columnSplit;
+
+ /**
+ * total number of columns
+ */
+ private int noOfColumnsStore;
+
+ /**
+ * whether given index is columnar or not
+ * true: columnar
+ * false: row block
+ */
+ private boolean[] columnarStore;
+
+ /**
+ * column groups
+ * e.g
+ * {{0,1,2},3,4,{5,6}}
+ */
+ private int[][] columnGroups;
+
+ /**
+ * return columnSplit
+ *
+ * @return
+ */
+ public int[] getColumnSplit() {
+ return columnSplit;
+ }
+
+ /**
+ * set columnSplit
+ *
+ * @param split
+ */
+ public void setColumnSplit(int[] split) {
+ this.columnSplit = split;
+ }
+
+ /**
+ * @return no of columnar block
+ */
+ public int getNoOfColumnStore() {
+ return this.noOfColumnsStore;
+ }
+
+ /**
+ * set no of columnar block
+ *
+ * @param noOfColumnsStore
+ */
+ public void setNoOfColumnStore(int noOfColumnsStore) {
+ this.noOfColumnsStore = noOfColumnsStore;
+ }
+
+ /**
+ * it's an identifier for row block or single column block
+ *
+ * @param columnarStore
+ */
+ public void setColumnarStore(boolean[] columnarStore) {
+ this.columnarStore = columnarStore;
+ }
+
+ /**
+ * set column groups
+ *
+ * @param columnGroups
+ */
+ public void setColumnGroup(int[][] columnGroups) {
+ this.columnGroups = columnGroups;
+ }
+
+ /**
+ * check if given column group is columnar
+ *
+ * @param colGroup
+ * @return true if given block is columnar
+ */
+ public boolean isColumnar(int colGroup) {
+ return columnarStore[colGroup];
+ }
+
+ /**
+ * @return columngroups
+ */
+ public int[][] getColumnGroup() {
+ return this.columnGroups;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java
new file mode 100644
index 0000000..4b35a7f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndex.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.util.Arrays;
+
+import org.apache.carbondata.core.util.ByteUtil;
+
+public class ColumnWithIntIndex implements Comparable<ColumnWithIntIndex> {
+ protected byte[] column;
+
+ private int index;
+
+ public ColumnWithIntIndex(byte[] column, int index) {
+ this.column = column;
+ this.index = index;
+ }
+
+ public ColumnWithIntIndex() {
+ }
+
+ /**
+ * @return the column
+ */
+ public byte[] getColumn() {
+ return column;
+ }
+
+ /**
+ * @param column the column to set
+ */
+ public void setColumn(byte[] column) {
+ this.column = column;
+ }
+
+ /**
+ * @return the index
+ */
+ public int getIndex() {
+ return index;
+ }
+
+ /**
+ * @param index the index to set
+ */
+ public void setIndex(int index) {
+ this.index = index;
+ }
+
+ @Override public int compareTo(ColumnWithIntIndex o) {
+ return ByteUtil.UnsafeComparer.INSTANCE.compareTo(column, o.column);
+ }
+
+ @Override public boolean equals(Object obj) {
+ if(obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ ColumnWithIntIndex o = (ColumnWithIntIndex)obj;
+ return Arrays.equals(column, o.column) && index == o.index;
+ }
+
+ @Override public int hashCode() {
+ return Arrays.hashCode(column) + index;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java
new file mode 100644
index 0000000..a6eb6c4
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnWithIntIndexForHighCard.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.util.Arrays;
+
+import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
+
+public class ColumnWithIntIndexForHighCard extends ColumnWithIntIndex
+ implements Comparable<ColumnWithIntIndex> {
+
+ public ColumnWithIntIndexForHighCard(byte[] column, int index) {
+ super(column, index);
+ }
+
+ @Override public int compareTo(ColumnWithIntIndex o) {
+ return UnsafeComparer.INSTANCE
+ .compareTo(column, 2, column.length - 2, o.column, 2, o.column.length - 2);
+ }
+
+ @Override public boolean equals(Object obj) {
+ if(obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ ColumnWithIntIndexForHighCard o = (ColumnWithIntIndexForHighCard)obj;
+ return Arrays.equals(column, o.column) && getIndex() == o.getIndex();
+ }
+
+ @Override public int hashCode() {
+ return Arrays.hashCode(column) + getIndex();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreDataHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreDataHolder.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreDataHolder.java
new file mode 100644
index 0000000..6c3acf9
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreDataHolder.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.nio.ByteBuffer;
+
+public class ColumnarKeyStoreDataHolder {
+ private byte[] keyblockData;
+ private ColumnarKeyStoreMetadata columnarKeyStoreMetadata;
+
+ public ColumnarKeyStoreDataHolder(final byte[] keyblockData,
+ final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) {
+ this.keyblockData = keyblockData;
+ this.columnarKeyStoreMetadata = columnarKeyStoreMetadata;
+ }
+
+ public ColumnarKeyStoreDataHolder(final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) {
+ this.columnarKeyStoreMetadata = columnarKeyStoreMetadata;
+ }
+
+ public int getSurrogateKey(int columnIndex) {
+ byte[] actual = new byte[4];
+ int startIndex;
+ if (null != columnarKeyStoreMetadata.getColumnReverseIndex()) {
+ startIndex =
+ columnarKeyStoreMetadata.getColumnReverseIndex()[columnIndex] * columnarKeyStoreMetadata
+ .getEachRowSize();
+ } else {
+ startIndex = columnIndex * columnarKeyStoreMetadata.getEachRowSize();
+ }
+ int destPos = 4 - columnarKeyStoreMetadata.getEachRowSize();
+ System.arraycopy(keyblockData, startIndex, actual, destPos,
+ columnarKeyStoreMetadata.getEachRowSize());
+ return ByteBuffer.wrap(actual).getInt();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreMetadata.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreMetadata.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreMetadata.java
new file mode 100644
index 0000000..706708f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreMetadata.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+class ColumnarKeyStoreMetadata {
+
+ private int[] columnReverseIndex;
+
+ private int eachRowSize;
+
+ ColumnarKeyStoreMetadata(int eachRowSize) {
+ this.eachRowSize = eachRowSize;
+ }
+
+ /**
+ * @return the eachRowSize
+ */
+ int getEachRowSize() {
+ return eachRowSize;
+ }
+
+ /**
+ * @return the columnReverseIndex
+ */
+ int[] getColumnReverseIndex() {
+ return columnReverseIndex;
+ }
+
+ /**
+ * @param columnReverseIndex the columnReverseIndex to set
+ */
+ void setColumnReverseIndex(int[] columnReverseIndex) {
+ this.columnReverseIndex = columnReverseIndex;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java
new file mode 100644
index 0000000..ac8f6b7
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/IndexStorage.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+public interface IndexStorage<T> {
+ boolean isAlreadySorted();
+
+ T getDataAfterComp();
+
+ T getIndexMap();
+
+ byte[][] getKeyBlock();
+
+ T getDataIndexMap();
+
+ int getTotalSize();
+
+ /**
+ * @return min value of block
+ */
+ byte[] getMin();
+
+ /**
+ * @return max value of block
+ */
+ byte[] getMax();
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/columnar/UnBlockIndexer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/UnBlockIndexer.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/UnBlockIndexer.java
new file mode 100644
index 0000000..4d4eac9
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/UnBlockIndexer.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.util.Arrays;
+
+public final class UnBlockIndexer {
+
+ private UnBlockIndexer() {
+
+ }
+
+ public static int[] uncompressIndex(int[] indexData, int[] indexMap) {
+ int actualSize = indexData.length;
+ for (int i = 0; i < indexMap.length; i++) {
+ actualSize += indexData[indexMap[i] + 1] - indexData[indexMap[i]] - 1;
+ }
+ int[] indexes = new int[actualSize];
+ int k = 0;
+ for (int i = 0; i < indexData.length; i++) {
+ int index = Arrays.binarySearch(indexMap, i);
+ if (index > -1) {
+ for (int j = indexData[indexMap[index]]; j <= indexData[indexMap[index] + 1]; j++) {
+ indexes[k] = j;
+ k++;
+ }
+ i++;
+ } else {
+ indexes[k] = indexData[i];
+ k++;
+ }
+ }
+ return indexes;
+ }
+
+ public static byte[] uncompressData(byte[] data, int[] index, int keyLen) {
+ if (index.length < 1) {
+ return data;
+ }
+ int numberOfCopy = 0;
+ int actualSize = 0;
+ int srcPos = 0;
+ int destPos = 0;
+ for (int i = 1; i < index.length; i += 2) {
+ actualSize += index[i];
+ }
+ byte[] uncompressedData = new byte[actualSize * keyLen];
+ int picIndex = 0;
+ for (int i = 0; i < data.length; i += keyLen) {
+ numberOfCopy = index[picIndex * 2 + 1];
+ picIndex++;
+ for (int j = 0; j < numberOfCopy; j++) {
+ System.arraycopy(data, srcPos, uncompressedData, destPos, keyLen);
+ destPos += keyLen;
+ }
+ srcPos += keyLen;
+ }
+ return uncompressedData;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/Compressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/Compressor.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/Compressor.java
new file mode 100644
index 0000000..d9803a0
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/Compressor.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression;
+
+public interface Compressor {
+
+ byte[] compressByte(byte[] unCompInput);
+
+ byte[] unCompressByte(byte[] compInput);
+
+ byte[] unCompressByte(byte[] compInput, int offset, int length);
+
+ byte[] compressShort(short[] unCompInput);
+
+ short[] unCompressShort(byte[] compInput);
+
+ short[] unCompressShort(byte[] compInput, int offset, int lenght);
+
+ byte[] compressInt(int[] unCompInput);
+
+ int[] unCompressInt(byte[] compInput);
+
+ int[] unCompressInt(byte[] compInput, int offset, int length);
+
+ byte[] compressLong(long[] unCompInput);
+
+ long[] unCompressLong(byte[] compInput);
+
+ long[] unCompressLong(byte[] compInput, int offset, int length);
+
+ byte[] compressFloat(float[] unCompInput);
+
+ float[] unCompressFloat(byte[] compInput);
+
+ float[] unCompressFloat(byte[] compInput, int offset, int length);
+
+ byte[] compressDouble(double[] unCompInput);
+
+ double[] unCompressDouble(byte[] compInput);
+
+ double[] unCompressDouble(byte[] compInput, int offset, int length);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/CompressorFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/CompressorFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/CompressorFactory.java
new file mode 100644
index 0000000..022ac97
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/CompressorFactory.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+public class CompressorFactory {
+
+ private static final CompressorFactory COMPRESSOR_FACTORY = new CompressorFactory();
+
+ private final Compressor compressor;
+
+ private CompressorFactory() {
+ String compressorType = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.COMPRESSOR, CarbonCommonConstants.DEFAULT_COMPRESSOR);
+ switch (compressorType) {
+ case "snappy":
+ compressor = new SnappyCompressor();
+ break;
+ default:
+ throw new RuntimeException(
+ "Invalid compressor type provided! Please provide valid compressor type");
+ }
+ }
+
+ public static CompressorFactory getInstance() {
+ return COMPRESSOR_FACTORY;
+ }
+
+ public Compressor getCompressor() {
+ return compressor;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java
new file mode 100644
index 0000000..1ded77d
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/MeasureMetaDataModel.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression;
+
+public class MeasureMetaDataModel {
+ /**
+ * maxValue
+ */
+ private Object[] maxValue;
+
+ /**
+ * minValue
+ */
+ private Object[] minValue;
+
+ /**
+ * mantissa
+ */
+ private int[] mantissa;
+
+ /**
+ * measureCount
+ */
+ private int measureCount;
+
+ /**
+ * uniqueValue
+ */
+ private Object[] uniqueValue;
+
+ /**
+ * type
+ */
+ private char[] type;
+
+ /**
+ * dataTypeSelected
+ */
+ private byte[] dataTypeSelected;
+
+ public MeasureMetaDataModel(Object[] minValue, Object[] maxValue, int[] mantissa,
+ int measureCount, Object[] uniqueValue, char[] type, byte[] dataTypeSelected) {
+ this.minValue = minValue;
+ this.maxValue = maxValue;
+ this.mantissa = mantissa;
+ this.measureCount = measureCount;
+ this.uniqueValue = uniqueValue;
+ this.type = type;
+ this.dataTypeSelected = dataTypeSelected;
+ }
+
+ /**
+ * get Max value
+ *
+ * @return
+ */
+ public Object[] getMaxValue() {
+ return maxValue;
+ }
+
+ /**
+ * getMinValue
+ *
+ * @return
+ */
+ public Object[] getMinValue() {
+ return minValue;
+ }
+
+ /**
+ * getMantissa
+ *
+ * @return
+ */
+ public int[] getMantissa() {
+ return mantissa;
+ }
+
+ /**
+ * getMeasureCount
+ *
+ * @return
+ */
+ public int getMeasureCount() {
+ return measureCount;
+ }
+
+ /**
+ * getUniqueValue
+ *
+ * @return
+ */
+ public Object[] getUniqueValue() {
+ return uniqueValue;
+ }
+
+ /**
+ * @return the type
+ */
+ public char[] getType() {
+ return type;
+ }
+
+ /**
+ * @return the dataTypeSelected
+ */
+ public byte[] getDataTypeSelected() {
+ return dataTypeSelected;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java
new file mode 100644
index 0000000..7a5c670
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ReaderCompressModel.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression;
+
+import org.apache.carbondata.core.metadata.ValueEncoderMeta;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+
+// Used in read path for decompression preparation
+public class ReaderCompressModel {
+ private ValueEncoderMeta valueEncoderMeta;
+
+ private ValueCompressionUtil.DataType convertedDataType;
+
+ private ValueCompressionHolder valueHolder;
+
+ public void setValueEncoderMeta(ValueEncoderMeta valueEncoderMeta) {
+ this.valueEncoderMeta = valueEncoderMeta;
+ }
+
+ public ValueCompressionUtil.DataType getConvertedDataType() {
+ return convertedDataType;
+ }
+
+ public void setConvertedDataType(ValueCompressionUtil.DataType convertedDataType) {
+ this.convertedDataType = convertedDataType;
+ }
+
+ public Object getMaxValue() {
+ return valueEncoderMeta.getMaxValue();
+ }
+
+ public int getMantissa() {
+ return valueEncoderMeta.getMantissa();
+ }
+
+ public ValueCompressionHolder getValueCompressionHolder() {
+ return valueHolder;
+ }
+
+ public void setValueCompressionHolder(ValueCompressionHolder valueHolder) {
+ this.valueHolder = valueHolder;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/SnappyCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/SnappyCompressor.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/SnappyCompressor.java
new file mode 100644
index 0000000..dd78bb1
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/SnappyCompressor.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+
+import org.xerial.snappy.Snappy;
+import org.xerial.snappy.SnappyNative;
+
+public class SnappyCompressor implements Compressor {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(SnappyCompressor.class.getName());
+
+ private final SnappyNative snappyNative;
+
+ public SnappyCompressor() {
+ Snappy snappy = new Snappy();
+ Field privateField = null;
+ try {
+ privateField = snappy.getClass().getDeclaredField("impl");
+ } catch (NoSuchFieldException | SecurityException e) {
+ throw new RuntimeException(e);
+ }
+ privateField.setAccessible(true);
+ try {
+ snappyNative = (SnappyNative) privateField.get(snappy);
+ } catch (IllegalArgumentException | IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override public byte[] compressByte(byte[] unCompInput) {
+ try {
+ return Snappy.rawCompress(unCompInput, unCompInput.length);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ return null;
+ }
+ }
+
+ @Override public byte[] unCompressByte(byte[] compInput) {
+ try {
+ return Snappy.uncompress(compInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return compInput;
+ }
+
+ @Override public byte[] unCompressByte(byte[] compInput, int offset, int length) {
+ int uncompressedLength = 0;
+ byte[] data = null;
+ try {
+ uncompressedLength = Snappy.uncompressedLength(compInput, offset, length);
+ data = new byte[uncompressedLength];
+ Snappy.uncompress(compInput, offset, length, data, 0);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return data;
+ }
+
+ @Override public byte[] compressShort(short[] unCompInput) {
+ try {
+ return Snappy.compress(unCompInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ return null;
+ }
+ }
+
+ @Override public short[] unCompressShort(byte[] compInput) {
+ try {
+ return Snappy.uncompressShortArray(compInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public short[] unCompressShort(byte[] compInput, int offset, int lenght) {
+ try {
+ return Snappy.uncompressShortArray(compInput, offset, lenght);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public byte[] compressInt(int[] unCompInput) {
+ try {
+ return Snappy.compress(unCompInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ return null;
+ }
+ }
+
+ @Override public int[] unCompressInt(byte[] compInput) {
+ try {
+ return Snappy.uncompressIntArray(compInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public int[] unCompressInt(byte[] compInput, int offset, int length) {
+ try {
+ return Snappy.uncompressIntArray(compInput, offset, length);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public byte[] compressLong(long[] unCompInput) {
+ try {
+ return Snappy.compress(unCompInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ return null;
+ }
+ }
+
+ @Override public long[] unCompressLong(byte[] compInput) {
+ try {
+ return Snappy.uncompressLongArray(compInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public long[] unCompressLong(byte[] compInput, int offset, int length) {
+ try {
+ return Snappy.uncompressLongArray(compInput, offset, length);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public byte[] compressFloat(float[] unCompInput) {
+ try {
+ return Snappy.compress(unCompInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ return null;
+ }
+ }
+
+ @Override public float[] unCompressFloat(byte[] compInput) {
+ try {
+ return Snappy.uncompressFloatArray(compInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public float[] unCompressFloat(byte[] compInput, int offset, int length) {
+ try {
+ return Snappy.uncompressFloatArray(compInput, offset, length);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public byte[] compressDouble(double[] unCompInput) {
+ try {
+ return Snappy.compress(unCompInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ return null;
+ }
+ }
+
+ @Override public double[] unCompressDouble(byte[] compInput) {
+ try {
+ return Snappy.uncompressDoubleArray(compInput);
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+
+ @Override public double[] unCompressDouble(byte[] compInput, int offset, int length) {
+ try {
+ int uncompressedLength = Snappy.uncompressedLength(compInput, offset, length);
+ double[] result = new double[uncompressedLength / 8];
+ snappyNative.rawUncompress(compInput, offset, length, result, 0);
+ return result;
+ } catch (IOException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java
new file mode 100644
index 0000000..8fed2ad
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/compression/ValueCompressionHolder.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.compression;
+
+import java.math.BigDecimal;
+
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+
+
+/**
+ * ValueCompressionHolder is the base class for handling
+ * compression / decompression of the measure data chunk
+ */
+public abstract class ValueCompressionHolder<T> {
+
+ /**
+ * compressedValue
+ */
+ protected byte[] compressedValue;
+
+ /**
+ * @param compressor the compressor used to decompress the data
+ * @param dataType data type of the data
+ * @param data compressed data
+ */
+ public void unCompress(Compressor compressor, DataType dataType, byte[] data,
+ int offset, int length) {
+ switch (dataType) {
+ case DATA_BYTE:
+ setValue((T)compressor.unCompressByte(data, offset, length));
+ break;
+ case DATA_SHORT:
+ setValue((T)compressor.unCompressShort(data, offset, length));
+ break;
+ case DATA_INT:
+ setValue((T)compressor.unCompressInt(data, offset, length));
+ break;
+ case DATA_LONG:
+ case DATA_BIGINT:
+ setValue((T)compressor.unCompressLong(data, offset, length));
+ break;
+ case DATA_FLOAT:
+ setValue((T)compressor.unCompressFloat(data, offset, length));
+ break;
+ default:
+ setValue((T)compressor.unCompressDouble(data, offset, length));
+ break;
+ }
+ }
+
+ /**
+ * @param compressor the compressor used to compress the data
+ * @param dataType data type of the data
+ * @param data original data
+ */
+ public byte[] compress(Compressor compressor, DataType dataType, Object data) {
+ switch (dataType) {
+ case DATA_BYTE:
+ return compressor.compressByte((byte[])data);
+ case DATA_SHORT:
+ return compressor.compressShort((short[])data);
+ case DATA_INT:
+ return compressor.compressInt((int[])data);
+ case DATA_LONG:
+ case DATA_BIGINT:
+ return compressor.compressLong((long[])data);
+ case DATA_FLOAT:
+ return compressor.compressFloat((float[])data);
+ case DATA_DOUBLE:
+ default:
+ return compressor.compressDouble((double[])data);
+ }
+ }
+
+ public abstract void setValue(T value);
+
+ public abstract T getValue();
+
+ public abstract void setValueInBytes(byte[] value);
+
+ public abstract void compress();
+
+ public abstract void uncompress(DataType dataType, byte[] compressData, int offset,
+ int length, int decimal, Object maxValueObject);
+
+ public byte[] getCompressedData() { return compressedValue; }
+
+ public abstract long getLongValue(int index);
+
+ public abstract double getDoubleValue(int index);
+
+ public abstract BigDecimal getBigDecimalValue(int index);
+
+ public abstract void freeMemory();
+
+
+}
[28/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/FileFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/FileFactory.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/FileFactory.java
deleted file mode 100644
index f0c424b..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/FileFactory.java
+++ /dev/null
@@ -1,485 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.impl;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.zip.GZIPInputStream;
-
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.core.datastorage.store.filesystem.AlluxioCarbonFile;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.filesystem.HDFSCarbonFile;
-import org.apache.carbondata.core.datastorage.store.filesystem.LocalCarbonFile;
-import org.apache.carbondata.core.datastorage.store.filesystem.ViewFSCarbonFile;
-import org.apache.carbondata.core.util.CarbonUtil;
-
-import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.compress.BZip2Codec;
-import org.apache.hadoop.io.compress.GzipCodec;
-
-public final class FileFactory {
- private static Configuration configuration = null;
-
- static {
- configuration = new Configuration();
- configuration.addResource(new Path("../core-default.xml"));
- }
-
- private FileFactory() {
-
- }
-
- public static Configuration getConfiguration() {
- return configuration;
- }
-
- public static FileHolder getFileHolder(FileType fileType) {
- switch (fileType) {
- case LOCAL:
- return new FileHolderImpl();
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- return new DFSFileHolderImpl();
- default:
- return new FileHolderImpl();
- }
- }
-
- public static FileType getFileType(String path) {
- if (path.startsWith(CarbonUtil.HDFS_PREFIX)) {
- return FileType.HDFS;
- }
- else if (path.startsWith(CarbonUtil.ALLUXIO_PREFIX)) {
- return FileType.ALLUXIO;
- }
- else if (path.startsWith(CarbonUtil.VIEWFS_PREFIX)) {
- return FileType.VIEWFS;
- }
- return FileType.LOCAL;
- }
-
- public static CarbonFile getCarbonFile(String path, FileType fileType) {
- switch (fileType) {
- case LOCAL:
- return new LocalCarbonFile(path);
- case HDFS:
- return new HDFSCarbonFile(path);
- case ALLUXIO:
- return new AlluxioCarbonFile(path);
- case VIEWFS:
- return new ViewFSCarbonFile(path);
- default:
- return new LocalCarbonFile(path);
- }
- }
-
- public static DataInputStream getDataInputStream(String path, FileType fileType)
- throws IOException {
- return getDataInputStream(path, fileType, -1);
- }
-
- public static DataInputStream getDataInputStream(String path, FileType fileType, int bufferSize)
- throws IOException {
- path = path.replace("\\", "/");
- boolean gzip = path.endsWith(".gz");
- boolean bzip2 = path.endsWith(".bz2");
- InputStream stream;
- switch (fileType) {
- case LOCAL:
- if (gzip) {
- stream = new GZIPInputStream(new FileInputStream(path));
- } else if (bzip2) {
- stream = new BZip2CompressorInputStream(new FileInputStream(path));
- } else {
- stream = new FileInputStream(path);
- }
- break;
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path pt = new Path(path);
- FileSystem fs = pt.getFileSystem(configuration);
- if (bufferSize == -1) {
- stream = fs.open(pt);
- } else {
- stream = fs.open(pt, bufferSize);
- }
- if (gzip) {
- GzipCodec codec = new GzipCodec();
- stream = codec.createInputStream(stream);
- } else if (bzip2) {
- BZip2Codec codec = new BZip2Codec();
- stream = codec.createInputStream(stream);
- }
- break;
- default:
- throw new UnsupportedOperationException("unsupported file system");
- }
- return new DataInputStream(new BufferedInputStream(stream));
- }
-
- /**
- * return the datainputStream which is seek to the offset of file
- *
- * @param path
- * @param fileType
- * @param bufferSize
- * @param offset
- * @return DataInputStream
- * @throws IOException
- */
- public static DataInputStream getDataInputStream(String path, FileType fileType, int bufferSize,
- long offset) throws IOException {
- path = path.replace("\\", "/");
- switch (fileType) {
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path pt = new Path(path);
- FileSystem fs = pt.getFileSystem(configuration);
- FSDataInputStream stream = fs.open(pt, bufferSize);
- stream.seek(offset);
- return new DataInputStream(new BufferedInputStream(stream));
- default:
- FileInputStream fis = new FileInputStream(path);
- long actualSkipSize = 0;
- long skipSize = offset;
- while (actualSkipSize != offset) {
- actualSkipSize += fis.skip(skipSize);
- skipSize = skipSize - actualSkipSize;
- }
- return new DataInputStream(new BufferedInputStream(fis));
- }
- }
-
- public static DataOutputStream getDataOutputStream(String path, FileType fileType)
- throws IOException {
- path = path.replace("\\", "/");
- switch (fileType) {
- case LOCAL:
- return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path)));
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path pt = new Path(path);
- FileSystem fs = pt.getFileSystem(configuration);
- FSDataOutputStream stream = fs.create(pt, true);
- return stream;
- default:
- return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path)));
- }
- }
-
- public static DataOutputStream getDataOutputStream(String path, FileType fileType, int bufferSize,
- boolean append) throws IOException {
- path = path.replace("\\", "/");
- switch (fileType) {
- case LOCAL:
- return new DataOutputStream(
- new BufferedOutputStream(new FileOutputStream(path, append), bufferSize));
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path pt = new Path(path);
- FileSystem fs = pt.getFileSystem(configuration);
- FSDataOutputStream stream = null;
- if (append) {
- // append to a file only if file already exists else file not found
- // exception will be thrown by hdfs
- if (CarbonUtil.isFileExists(path)) {
- stream = fs.append(pt, bufferSize);
- } else {
- stream = fs.create(pt, true, bufferSize);
- }
- } else {
- stream = fs.create(pt, true, bufferSize);
- }
- return stream;
- default:
- return new DataOutputStream(
- new BufferedOutputStream(new FileOutputStream(path), bufferSize));
- }
- }
-
- public static DataOutputStream getDataOutputStream(String path, FileType fileType, int bufferSize,
- long blockSize) throws IOException {
- path = path.replace("\\", "/");
- switch (fileType) {
- case LOCAL:
- return new DataOutputStream(
- new BufferedOutputStream(new FileOutputStream(path), bufferSize));
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path pt = new Path(path);
- FileSystem fs = pt.getFileSystem(configuration);
- FSDataOutputStream stream =
- fs.create(pt, true, bufferSize, fs.getDefaultReplication(pt), blockSize);
- return stream;
- default:
- return new DataOutputStream(
- new BufferedOutputStream(new FileOutputStream(path), bufferSize));
- }
- }
-
- /**
- * This method checks the given path exists or not and also is it file or
- * not if the performFileCheck is true
- *
- * @param filePath - Path
- * @param fileType - FileType Local/HDFS
- * @param performFileCheck - Provide false for folders, true for files and
- */
- public static boolean isFileExist(String filePath, FileType fileType, boolean performFileCheck)
- throws IOException {
- filePath = filePath.replace("\\", "/");
- switch (fileType) {
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path path = new Path(filePath);
- FileSystem fs = path.getFileSystem(configuration);
- if (performFileCheck) {
- return fs.exists(path) && fs.isFile(path);
- } else {
- return fs.exists(path);
- }
-
- case LOCAL:
- default:
- File defaultFile = new File(filePath);
-
- if (performFileCheck) {
- return defaultFile.exists() && defaultFile.isFile();
- } else {
- return defaultFile.exists();
- }
- }
- }
-
- /**
- * This method checks the given path exists or not and also is it file or
- * not if the performFileCheck is true
- *
- * @param filePath - Path
- * @param fileType - FileType Local/HDFS
- */
- public static boolean isFileExist(String filePath, FileType fileType) throws IOException {
- filePath = filePath.replace("\\", "/");
- switch (fileType) {
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path path = new Path(filePath);
- FileSystem fs = path.getFileSystem(configuration);
- return fs.exists(path);
-
- case LOCAL:
- default:
- File defaultFile = new File(filePath);
- return defaultFile.exists();
- }
- }
-
- public static boolean createNewFile(String filePath, FileType fileType) throws IOException {
- filePath = filePath.replace("\\", "/");
- switch (fileType) {
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path path = new Path(filePath);
- FileSystem fs = path.getFileSystem(configuration);
- return fs.createNewFile(path);
-
- case LOCAL:
- default:
- File file = new File(filePath);
- return file.createNewFile();
- }
- }
-
- public static boolean deleteFile(String filePath, FileType fileType) throws IOException {
- filePath = filePath.replace("\\", "/");
- switch (fileType) {
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path path = new Path(filePath);
- FileSystem fs = path.getFileSystem(configuration);
- return fs.delete(path, true);
-
- case LOCAL:
- default:
- File file = new File(filePath);
- return deleteAllFilesOfDir(file);
- }
- }
-
- public static boolean deleteAllFilesOfDir(File path) {
- if (!path.exists()) {
- return true;
- }
- if (path.isFile()) {
- return path.delete();
- }
- File[] files = path.listFiles();
- for (int i = 0; i < files.length; i++) {
- deleteAllFilesOfDir(files[i]);
- }
- return path.delete();
- }
-
- public static boolean mkdirs(String filePath, FileType fileType) throws IOException {
- filePath = filePath.replace("\\", "/");
- switch (fileType) {
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path path = new Path(filePath);
- FileSystem fs = path.getFileSystem(configuration);
- return fs.mkdirs(path);
- case LOCAL:
- default:
- File file = new File(filePath);
- return file.mkdirs();
- }
- }
-
- /**
- * for getting the dataoutput stream using the hdfs filesystem append API.
- *
- * @param path
- * @param fileType
- * @return
- * @throws IOException
- */
- public static DataOutputStream getDataOutputStreamUsingAppend(String path, FileType fileType)
- throws IOException {
- path = path.replace("\\", "/");
- switch (fileType) {
- case LOCAL:
- return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path, true)));
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path pt = new Path(path);
- FileSystem fs = pt.getFileSystem(configuration);
- FSDataOutputStream stream = fs.append(pt);
- return stream;
- default:
- return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path)));
- }
- }
-
- /**
- * for creating a new Lock file and if it is successfully created
- * then in case of abrupt shutdown then the stream to that file will be closed.
- *
- * @param filePath
- * @param fileType
- * @return
- * @throws IOException
- */
- public static boolean createNewLockFile(String filePath, FileType fileType) throws IOException {
- filePath = filePath.replace("\\", "/");
- switch (fileType) {
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path path = new Path(filePath);
- FileSystem fs = path.getFileSystem(configuration);
- if (fs.createNewFile(path)) {
- fs.deleteOnExit(path);
- return true;
- }
- return false;
- case LOCAL:
- default:
- File file = new File(filePath);
- return file.createNewFile();
- }
- }
-
- public enum FileType {
- LOCAL, HDFS, ALLUXIO, VIEWFS
- }
-
- /**
- * below method will be used to update the file path
- * for local type
- * it removes the file:/ from the path
- *
- * @param filePath
- * @return updated file path without url for local
- */
- public static String getUpdatedFilePath(String filePath) {
- FileType fileType = getFileType(filePath);
- switch (fileType) {
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- return filePath;
- case LOCAL:
- default:
- Path pathWithoutSchemeAndAuthority =
- Path.getPathWithoutSchemeAndAuthority(new Path(filePath));
- return pathWithoutSchemeAndAuthority.toString();
- }
- }
-
- /**
- * It computes size of directory
- *
- * @param filePath
- * @return size in bytes
- * @throws IOException
- */
- public static long getDirectorySize(String filePath) throws IOException {
- FileType fileType = getFileType(filePath);
- switch (fileType) {
- case HDFS:
- case ALLUXIO:
- case VIEWFS:
- Path path = new Path(filePath);
- FileSystem fs = path.getFileSystem(configuration);
- return fs.getContentSummary(path).getLength();
- case LOCAL:
- default:
- File file = new File(filePath);
- return FileUtils.sizeOfDirectory(file);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/FileHolderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/FileHolderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/FileHolderImpl.java
deleted file mode 100644
index 9a5c06a..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/FileHolderImpl.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.impl;
-
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-
-public class FileHolderImpl implements FileHolder {
- /**
- * cache to hold filename and its stream
- */
- private Map<String, FileChannel> fileNameAndStreamCache;
-
- /**
- * FileHolderImpl Constructor
- * It will create the cache
- */
- public FileHolderImpl() {
- this.fileNameAndStreamCache =
- new HashMap<String, FileChannel>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- }
-
- public FileHolderImpl(int capacity) {
- this.fileNameAndStreamCache = new HashMap<String, FileChannel>(capacity);
- }
-
- /**
- * This method will be used to read the byte array from file based on offset
- * and length(number of bytes) need to read
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @param length number of bytes to be read
- * @return read byte array
- */
- @Override public byte[] readByteArray(String filePath, long offset, int length)
- throws IOException {
- FileChannel fileChannel = updateCache(filePath);
- ByteBuffer byteBffer = read(fileChannel, length, offset);
- return byteBffer.array();
- }
-
- /**
- * This method will be used to close all the streams currently present in the cache
- */
- @Override public void finish() throws IOException {
- for (Entry<String, FileChannel> entry : fileNameAndStreamCache.entrySet()) {
- FileChannel channel = entry.getValue();
- if (null != channel) {
- channel.close();
- }
- }
- }
-
- /**
- * This method will be used to read int from file from postion(offset), here
- * length will be always 4 bacause int byte size if 4
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @return read int
- */
- @Override public int readInt(String filePath, long offset) throws IOException {
- FileChannel fileChannel = updateCache(filePath);
- ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.INT_SIZE_IN_BYTE, offset);
- return byteBffer.getInt();
- }
-
- /**
- * This method will be used to read int from file from postion(offset), here
- * length will be always 4 bacause int byte size if 4
- *
- * @param filePath fully qualified file path
- * @return read int
- */
- @Override public int readInt(String filePath) throws IOException {
- FileChannel fileChannel = updateCache(filePath);
- ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.INT_SIZE_IN_BYTE);
- return byteBffer.getInt();
- }
-
- /**
- * This method will be used to read int from file from postion(offset), here
- * length will be always 4 bacause int byte size if 4
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @return read int
- */
- @Override public long readDouble(String filePath, long offset) throws IOException {
- FileChannel fileChannel = updateCache(filePath);
- ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.LONG_SIZE_IN_BYTE, offset);
- return byteBffer.getLong();
- }
-
- /**
- * This method will be used to check whether stream is already present in
- * cache or not for filepath if not present then create it and then add to
- * cache, other wise get from cache
- *
- * @param filePath fully qualified file path
- * @return channel
- */
- private FileChannel updateCache(String filePath) throws FileNotFoundException {
- FileChannel fileChannel = fileNameAndStreamCache.get(filePath);
- if (null == fileChannel) {
- FileInputStream stream = new FileInputStream(filePath);
- fileChannel = stream.getChannel();
- fileNameAndStreamCache.put(filePath, fileChannel);
- }
- return fileChannel;
- }
-
- /**
- * This method will be used to read from file based on number of bytes to be read and positon
- *
- * @param channel file channel
- * @param size number of bytes
- * @param offset position
- * @return byte buffer
- */
- private ByteBuffer read(FileChannel channel, int size, long offset) throws IOException {
- ByteBuffer byteBffer = ByteBuffer.allocate(size);
- channel.position(offset);
- channel.read(byteBffer);
- byteBffer.rewind();
- return byteBffer;
- }
-
- /**
- * This method will be used to read from file based on number of bytes to be read and positon
- *
- * @param channel file channel
- * @param size number of bytes
- * @return byte buffer
- */
- private ByteBuffer read(FileChannel channel, int size) throws IOException {
- ByteBuffer byteBffer = ByteBuffer.allocate(size);
- channel.read(byteBffer);
- byteBffer.rewind();
- return byteBffer;
- }
-
-
- /**
- * This method will be used to read the byte array from file based on length(number of bytes)
- *
- * @param filePath fully qualified file path
- * @param length number of bytes to be read
- * @return read byte array
- */
- @Override public byte[] readByteArray(String filePath, int length) throws IOException {
- FileChannel fileChannel = updateCache(filePath);
- ByteBuffer byteBffer = read(fileChannel, length);
- return byteBffer.array();
- }
-
- /**
- * This method will be used to read long from file from postion(offset), here
- * length will be always 8 bacause int byte size is 8
- *
- * @param filePath fully qualified file path
- * @param offset reading start position,
- * @return read long
- */
- @Override public long readLong(String filePath, long offset) throws IOException {
- FileChannel fileChannel = updateCache(filePath);
- ByteBuffer byteBffer = read(fileChannel, CarbonCommonConstants.LONG_SIZE_IN_BYTE, offset);
- return byteBffer.getLong();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
deleted file mode 100644
index ad1d62f..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.impl.data.compressed;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.NodeMeasureDataStore;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-
-public abstract class AbstractHeavyCompressedDoubleArrayDataStore
- implements NodeMeasureDataStore //NodeMeasureDataStore<double[]>
-{
-
- private LogService LOGGER =
- LogServiceFactory.getLogService(AbstractHeavyCompressedDoubleArrayDataStore.class.getName());
-
- /**
- * values.
- */
- protected ValueCompressionHolder[] values;
-
- /**
- * compressionModel.
- */
- protected WriterCompressModel compressionModel;
-
- /**
- * type
- */
- private char[] type;
-
- /**
- * AbstractHeavyCompressedDoubleArrayDataStore constructor.
- *
- * @param compressionModel
- */
- public AbstractHeavyCompressedDoubleArrayDataStore(WriterCompressModel compressionModel) {
- this.compressionModel = compressionModel;
- if (null != compressionModel) {
- this.type = compressionModel.getType();
- values =
- new ValueCompressionHolder[compressionModel.getValueCompressionHolder().length];
- }
- }
-
- // this method first invokes encoding routine to encode the data chunk,
- // followed by invoking compression routine for preparing the data chunk for writing.
- @Override public byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] dataHolder) {
- byte[][] returnValue = new byte[values.length][];
- for (int i = 0; i < compressionModel.getValueCompressionHolder().length; i++) {
- values[i] = compressionModel.getValueCompressionHolder()[i];
- if (type[i] != CarbonCommonConstants.BYTE_VALUE_MEASURE) {
- // first perform encoding of the data chunk
- values[i].setValue(
- ValueCompressionUtil.getValueCompressor(compressionModel.getCompressionFinders()[i])
- .getCompressedValues(compressionModel.getCompressionFinders()[i], dataHolder[i],
- compressionModel.getMaxValue()[i],
- compressionModel.getMantissa()[i]));
- } else {
- values[i].setValue(dataHolder[i].getWritableByteArrayValues());
- }
- values[i].compress();
- returnValue[i] = values[i].getCompressedData();
- }
-
- return returnValue;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
deleted file mode 100644
index 0c29143..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.impl.data.compressed;
-
-import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
-
-public class HeavyCompressedDoubleArrayDataInMemoryStore
- extends AbstractHeavyCompressedDoubleArrayDataStore {
-
- public HeavyCompressedDoubleArrayDataInMemoryStore(WriterCompressModel compressionModel) {
- super(compressionModel);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/util/StoreFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/util/StoreFactory.java b/core/src/main/java/org/apache/carbondata/core/datastorage/util/StoreFactory.java
deleted file mode 100644
index 7561ec7..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/util/StoreFactory.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.util;
-
-import org.apache.carbondata.core.datastorage.store.NodeMeasureDataStore;
-import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastorage.store.impl.data.compressed.HeavyCompressedDoubleArrayDataInMemoryStore;
-
-public final class StoreFactory {
-
- private StoreFactory() {
- }
-
- public static NodeMeasureDataStore createDataStore(WriterCompressModel compressionModel) {
- return new HeavyCompressedDoubleArrayDataInMemoryStore(compressionModel);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java
index a80661c..68d0e95 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java
@@ -46,9 +46,9 @@ import org.apache.carbondata.core.datastore.block.TableBlockUniqueIdentifier;
import org.apache.carbondata.core.datastore.exception.IndexBuilderException;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.update.CarbonUpdateUtil;
-import org.apache.carbondata.core.update.UpdateVO;
+import org.apache.carbondata.core.update.data.UpdateVO;
import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.scan.model.QueryModel;
+import org.apache.carbondata.core.scan.model.QueryModel;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
index 81d80ef..219ee73 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
@@ -20,10 +20,10 @@ package org.apache.carbondata.core.datastore;
import java.io.IOException;
-import org.apache.carbondata.common.iudprocessor.cache.BlockletLevelDeleteDeltaDataCache;
+import org.apache.carbondata.core.update.data.BlockletLevelDeleteDeltaDataCache;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
-import org.apache.carbondata.core.datastorage.store.FileHolder;
+import org.apache.carbondata.core.datastorage.FileHolder;
/**
* Interface data block reference
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
index 03765a4..1ae8d28 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
@@ -41,7 +41,7 @@ import org.apache.carbondata.core.datastore.exception.IndexBuilderException;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.path.CarbonTablePath.DataFileUtil;
-import org.apache.carbondata.core.update.UpdateVO;
+import org.apache.carbondata.core.update.data.UpdateVO;
import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager;
import org.apache.carbondata.core.util.CarbonUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
index 2776fd5..e84c8ba 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
@@ -34,7 +34,7 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.columnar.ColumnGroupModel;
+import org.apache.carbondata.core.datastorage.columnar.ColumnGroupModel;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.keygenerator.columnar.ColumnarSplitter;
import org.apache.carbondata.core.keygenerator.columnar.impl.MultiDimKeyVarLengthVariableSplitGenerator;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java
index 383dba2..dc4c938 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java
@@ -25,7 +25,7 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.carbondata.core.cache.Cacheable;
import org.apache.carbondata.core.datastore.SegmentTaskIndexStore;
-import org.apache.carbondata.core.update.UpdateVO;
+import org.apache.carbondata.core.update.data.UpdateVO;
/**
* SegmentTaskIndexWrapper class holds the taskIdToTableSegmentMap
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
index 37c1aa0..a5061d2 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
@@ -26,7 +26,7 @@ import org.apache.carbondata.core.ColumnarFormatVersion;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.path.CarbonTablePath.DataFileUtil;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
/**
* class will be used to pass the block detail detail will be passed form driver
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnDataChunk.java
index 731bf95..086785a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/DimensionColumnDataChunk.java
@@ -18,8 +18,8 @@
*/
package org.apache.carbondata.core.datastore.chunk;
-import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.scan.result.vector.ColumnVectorInfo;
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
/**
* Interface for dimension column chunk.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java
index 285f66e..d2ee1a7 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java
@@ -19,7 +19,7 @@
package org.apache.carbondata.core.datastore.chunk;
import org.apache.carbondata.core.metadata.blocklet.datachunk.PresenceMeta;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder;
/**
* Holder for measure column chunk
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
index 08875c2..bad5679 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
@@ -20,8 +20,8 @@ package org.apache.carbondata.core.datastore.chunk.impl;
import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
-import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.scan.result.vector.ColumnVectorInfo;
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
/**
* This class is gives access to column group dimension data chunk store
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
index c1f668b..9eee078 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
@@ -21,9 +21,9 @@ package org.apache.carbondata.core.datastore.chunk.impl;
import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.scan.result.vector.CarbonColumnVector;
-import org.apache.carbondata.scan.result.vector.ColumnVectorInfo;
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
+import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
/**
* This class is gives access to fixed length dimension data chunk store
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
index 5084e38..7c16f54 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
@@ -23,9 +23,9 @@ import java.util.Arrays;
import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.scan.result.vector.CarbonColumnVector;
-import org.apache.carbondata.scan.result.vector.ColumnVectorInfo;
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
+import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
/**
* This class is gives access to variable length dimension data chunk store
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
index 0722df4..aeb7ff9 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
@@ -21,7 +21,7 @@ package org.apache.carbondata.core.datastore.chunk.reader;
import java.io.IOException;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastorage.store.FileHolder;
+import org.apache.carbondata.core.datastorage.FileHolder;
/**
* Interface for reading the data chunk
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
index 7d3f891..ec86612 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
@@ -21,7 +21,7 @@ package org.apache.carbondata.core.datastore.chunk.reader;
import java.io.IOException;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
-import org.apache.carbondata.core.datastorage.store.FileHolder;
+import org.apache.carbondata.core.datastorage.FileHolder;
/**
* Reader interface for reading the measure blocks from file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java
index 450dc58..4e45bc3 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java
@@ -20,8 +20,8 @@ package org.apache.carbondata.core.datastore.chunk.reader.dimension;
import org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
+import org.apache.carbondata.core.datastorage.compression.Compressor;
+import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
import org.apache.carbondata.core.util.CarbonProperties;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
index 2db8667..01df68b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
@@ -29,8 +29,8 @@ import org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunk
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk;
import org.apache.carbondata.core.metadata.Encoding;
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.core.datastorage.store.columnar.UnBlockIndexer;
+import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.datastorage.columnar.UnBlockIndexer;
import org.apache.carbondata.core.util.CarbonUtil;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
index 07baed3..48b5aeb 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
@@ -27,8 +27,8 @@ import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataC
import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
import org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReader;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.core.datastorage.store.columnar.UnBlockIndexer;
+import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.datastorage.columnar.UnBlockIndexer;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.format.DataChunk2;
import org.apache.carbondata.format.Encoding;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
index c09b7e9..562ea8c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
@@ -25,10 +25,10 @@ import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReader;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk;
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.core.datastorage.store.compression.ReaderCompressModel;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.datastorage.compression.ReaderCompressModel;
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.core.util.ValueCompressionUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
index 9e3db25..90fa6f3 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
@@ -27,11 +27,11 @@ import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReader;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.metadata.blocklet.datachunk.PresenceMeta;
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.format.DataChunk2;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
index 6fdf5d0..14df51f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
@@ -20,12 +20,12 @@ package org.apache.carbondata.core.datastore.impl.btree;
import java.io.IOException;
-import org.apache.carbondata.common.iudprocessor.cache.BlockletLevelDeleteDeltaDataCache;
+import org.apache.carbondata.core.update.data.BlockletLevelDeleteDeltaDataCache;
import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
-import org.apache.carbondata.core.datastorage.store.FileHolder;
+import org.apache.carbondata.core.datastorage.FileHolder;
/**
* Non leaf node abstract class
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
index 9332b93..c2ffae9 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
@@ -21,13 +21,13 @@ package org.apache.carbondata.core.datastore.impl.btree;
import java.util.ArrayList;
import java.util.List;
-import org.apache.carbondata.common.iudprocessor.cache.BlockletLevelDeleteDeltaDataCache;
+import org.apache.carbondata.core.update.data.BlockletLevelDeleteDeltaDataCache;
import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.FileHolder;
+import org.apache.carbondata.core.datastorage.FileHolder;
/**
* No leaf node of a b+tree class which will keep the matadata(start key) of the
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
index e9a7a1a..f3d2bef 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
@@ -27,7 +27,7 @@ import org.apache.carbondata.core.datastore.chunk.reader.CarbonDataReaderFactory
import org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader;
import org.apache.carbondata.core.datastore.chunk.reader.MeasureColumnChunkReader;
import org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex;
-import org.apache.carbondata.core.datastorage.store.FileHolder;
+import org.apache.carbondata.core.datastorage.FileHolder;
/**
* Leaf node class of a Blocklet btree
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java
index 47ca673..9df57b0 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java
@@ -22,7 +22,7 @@ import java.io.IOException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
-import org.apache.carbondata.common.factory.CarbonCommonFactory;
+import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.Cache;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/load/BlockDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/load/BlockDetails.java b/core/src/main/java/org/apache/carbondata/core/load/BlockDetails.java
index 69c7cf5..5f00f23 100644
--- a/core/src/main/java/org/apache/carbondata/core/load/BlockDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/load/BlockDetails.java
@@ -21,7 +21,7 @@ package org.apache.carbondata.core.load;
import java.io.Serializable;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfoColumnar.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfoColumnar.java b/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfoColumnar.java
index c42bc93..1f3090d 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfoColumnar.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfoColumnar.java
@@ -21,7 +21,7 @@ package org.apache.carbondata.core.metadata;
import java.util.BitSet;
-import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
public class BlockletInfoColumnar {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/partition/Partitioner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/partition/Partitioner.java b/core/src/main/java/org/apache/carbondata/core/partition/Partitioner.java
deleted file mode 100644
index 1907687..0000000
--- a/core/src/main/java/org/apache/carbondata/core/partition/Partitioner.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.partition;
-
-/**
- * Partitions the data as per key
- */
-public interface Partitioner<Key> {
-
- int getPartition(Key key);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/partition/impl/HashPartitionerImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/partition/impl/HashPartitionerImpl.java b/core/src/main/java/org/apache/carbondata/core/partition/impl/HashPartitionerImpl.java
deleted file mode 100644
index 18936d6..0000000
--- a/core/src/main/java/org/apache/carbondata/core/partition/impl/HashPartitionerImpl.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.partition.impl;
-
-import java.util.List;
-
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.partition.Partitioner;
-
-/**
- * Hash partitioner implementation
- */
-public class HashPartitionerImpl implements Partitioner<Object[]> {
-
- private int numberOfBuckets;
-
- private Hash[] hashes;
-
- public HashPartitionerImpl(List<Integer> indexes, List<ColumnSchema> columnSchemas,
- int numberOfBuckets) {
- this.numberOfBuckets = numberOfBuckets;
- hashes = new Hash[indexes.size()];
- for (int i = 0; i < indexes.size(); i++) {
- switch(columnSchemas.get(i).getDataType()) {
- case SHORT:
- case INT:
- case LONG:
- hashes[i] = new IntegralHash(indexes.get(i));
- break;
- case DOUBLE:
- case FLOAT:
- case DECIMAL:
- hashes[i] = new DecimalHash(indexes.get(i));
- break;
- default:
- hashes[i] = new StringHash(indexes.get(i));
- }
- }
- }
-
- @Override public int getPartition(Object[] objects) {
- int hashCode = 0;
- for (Hash hash : hashes) {
- hashCode += hash.getHash(objects);
- }
- return (hashCode & Integer.MAX_VALUE) % numberOfBuckets;
- }
-
- private interface Hash {
- int getHash(Object[] value);
- }
-
- private static class IntegralHash implements Hash {
-
- private int index;
-
- private IntegralHash(int index) {
- this.index = index;
- }
-
- public int getHash(Object[] value) {
- return value[index] != null ? Long.valueOf(value[index].toString()).hashCode() : 0;
- }
- }
-
- private static class DecimalHash implements Hash {
-
- private int index;
-
- private DecimalHash(int index) {
- this.index = index;
- }
-
- public int getHash(Object[] value) {
- return value[index] != null ? Double.valueOf(value[index].toString()).hashCode() : 0;
- }
- }
-
- private static class StringHash implements Hash {
-
- private int index;
-
- private StringHash(int index) {
- this.index = index;
- }
-
- @Override public int getHash(Object[] value) {
- return value[index] != null ? value[index].hashCode() : 0;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/path/CarbonTablePath.java
index 337e2bb..b762ca1 100644
--- a/core/src/main/java/org/apache/carbondata/core/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/path/CarbonTablePath.java
@@ -22,9 +22,9 @@ import java.io.File;
import org.apache.carbondata.core.CarbonTableIdentifier;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import static org.apache.carbondata.core.constants.CarbonCommonConstants.INVALID_SEGMENT_ID;
import org.apache.hadoop.fs.Path;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java
index dac2d43..b76353c 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java
@@ -24,7 +24,7 @@ import java.io.*;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.update.DeleteDeltaBlockDetails;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.fileoperations.AtomicFileOperations;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteFilesDataReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteFilesDataReader.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteFilesDataReader.java
index a0922ec..04f6dfd 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteFilesDataReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteFilesDataReader.java
@@ -29,7 +29,7 @@ import java.util.concurrent.*;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.update.DeleteDeltaBlockDetails;
import org.apache.carbondata.core.update.DeleteDeltaBlockletDetails;
import org.apache.carbondata.core.util.CarbonProperties;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
index 9bf1919..43ea57f 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
@@ -23,7 +23,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.carbondata.common.factory.CarbonCommonFactory;
+import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.core.CarbonTableIdentifier;
import org.apache.carbondata.core.ColumnIdentifier;
import org.apache.carbondata.core.path.CarbonTablePath;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java
index ea87ed1..27b1fc3 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java
@@ -25,7 +25,7 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import org.apache.carbondata.common.factory.CarbonCommonFactory;
+import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.core.cache.dictionary.ColumnDictionaryChunkIterator;
import org.apache.carbondata.core.CarbonTableIdentifier;
import org.apache.carbondata.core.ColumnIdentifier;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/reader/ThriftReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/ThriftReader.java b/core/src/main/java/org/apache/carbondata/core/reader/ThriftReader.java
index 81b7661..8a7abd6 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/ThriftReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/ThriftReader.java
@@ -22,7 +22,7 @@ package org.apache.carbondata.core.reader;
import java.io.DataInputStream;
import java.io.IOException;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.thrift.TBase;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
index 99adbb4..1dd51d4 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
@@ -21,13 +21,13 @@ package org.apache.carbondata.core.reader.sortindex;
import java.io.IOException;
import java.util.List;
-import org.apache.carbondata.common.factory.CarbonCommonFactory;
+import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.CarbonTableIdentifier;
import org.apache.carbondata.core.ColumnIdentifier;
import org.apache.carbondata.core.path.CarbonTablePath;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk;
import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader;
import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReaderImpl;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/collector/ScannedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/ScannedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/ScannedResultCollector.java
new file mode 100644
index 0000000..7ea0625
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/ScannedResultCollector.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.collector;
+
+import java.util.List;
+
+import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
+
+/**
+ * Interface which will be used to aggregate the scan result
+ */
+public interface ScannedResultCollector {
+
+ /**
+ * Below method will be used to aggregate the scanned result
+ *
+ * @param scannedResult scanned result
+ * @return how many records was aggregated
+ */
+ List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize);
+
+ /**
+ * Collects data in columnar format.
+ * @param scannedResult
+ * @param columnarBatch
+ */
+ void collectVectorBatch(AbstractScannedResult scannedResult, CarbonColumnarBatch columnarBatch);
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
new file mode 100644
index 0000000..3f7dd4a
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.collector.impl;
+
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.scan.collector.ScannedResultCollector;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.scan.executor.util.QueryUtil;
+import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
+import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
+
+/**
+ * It is not a collector it is just a scanned result holder.
+ */
+public abstract class AbstractScannedResultCollector implements ScannedResultCollector {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(AbstractScannedResultCollector.class.getName());
+
+ /**
+ * restructuring info
+ */
+ private KeyStructureInfo restructureInfos;
+
+ /**
+ * table block execution infos
+ */
+ protected BlockExecutionInfo tableBlockExecutionInfos;
+
+ /**
+ * Measure ordinals
+ */
+ protected int[] measuresOrdinal;
+
+ /**
+ * to check whether measure exists in current table block or not this to
+ * handle restructuring scenario
+ */
+ protected boolean[] isMeasureExistsInCurrentBlock;
+
+ /**
+ * default value of the measures in case of restructuring some measure wont
+ * be present in the table so in that default value will be used to
+ * aggregate the data for that measure columns
+ */
+ private Object[] measureDefaultValue;
+
+ /**
+ * measure datatypes.
+ */
+ protected DataType[] measureDatatypes;
+
+ public AbstractScannedResultCollector(BlockExecutionInfo blockExecutionInfos) {
+ this.tableBlockExecutionInfos = blockExecutionInfos;
+ restructureInfos = blockExecutionInfos.getKeyStructureInfo();
+ measuresOrdinal = tableBlockExecutionInfos.getAggregatorInfo().getMeasureOrdinals();
+ isMeasureExistsInCurrentBlock = tableBlockExecutionInfos.getAggregatorInfo().getMeasureExists();
+ measureDefaultValue = tableBlockExecutionInfos.getAggregatorInfo().getDefaultValues();
+ this.measureDatatypes = tableBlockExecutionInfos.getAggregatorInfo().getMeasureDataTypes();
+ }
+
+ protected void fillMeasureData(Object[] msrValues, int offset,
+ AbstractScannedResult scannedResult) {
+ for (short i = 0; i < measuresOrdinal.length; i++) {
+ // if measure exists is block then pass measure column
+ // data chunk to the collector
+ if (isMeasureExistsInCurrentBlock[i]) {
+ msrValues[i + offset] = getMeasureData(scannedResult.getMeasureChunk(measuresOrdinal[i]),
+ scannedResult.getCurrenrRowId(), measureDatatypes[i]);
+ } else {
+ // if not then get the default value and use that value in aggregation
+ msrValues[i + offset] = measureDefaultValue[i];
+ }
+ }
+ }
+
+ private Object getMeasureData(MeasureColumnDataChunk dataChunk, int index, DataType dataType) {
+ if (!dataChunk.getNullValueIndexHolder().getBitSet().get(index)) {
+ switch (dataType) {
+ case SHORT:
+ case INT:
+ case LONG:
+ return dataChunk.getMeasureDataHolder().getReadableLongValueByIndex(index);
+ case DECIMAL:
+ return org.apache.spark.sql.types.Decimal
+ .apply(dataChunk.getMeasureDataHolder().getReadableBigDecimalValueByIndex(index));
+ default:
+ return dataChunk.getMeasureDataHolder().getReadableDoubleValueByIndex(index);
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Below method will used to get the result
+ */
+ protected void updateData(List<Object[]> listBasedResult) {
+ if (tableBlockExecutionInfos.isFixedKeyUpdateRequired()) {
+ updateKeyWithLatestBlockKeygenerator(listBasedResult);
+ }
+ }
+
+ /**
+ * Below method will be used to update the fixed length key with the
+ * latest block key generator
+ *
+ * @return updated block
+ */
+ private void updateKeyWithLatestBlockKeygenerator(List<Object[]> listBasedResult) {
+ try {
+ long[] data = null;
+ ByteArrayWrapper key = null;
+ for (int i = 0; i < listBasedResult.size(); i++) {
+ // get the key
+ key = (ByteArrayWrapper) listBasedResult.get(i)[0];
+ // unpack the key with table block key generator
+ data = tableBlockExecutionInfos.getBlockKeyGenerator()
+ .getKeyArray(key.getDictionaryKey(), tableBlockExecutionInfos.getMaskedByteForBlock());
+ // packed the key with latest block key generator
+ // and generate the masked key for that key
+ key.setDictionaryKey(QueryUtil
+ .getMaskedKey(restructureInfos.getKeyGenerator().generateKey(data),
+ restructureInfos.getMaxKey(), restructureInfos.getMaskByteRanges(),
+ restructureInfos.getMaskByteRanges().length));
+ }
+ } catch (KeyGenException e) {
+ LOGGER.error(e);
+ }
+ }
+
+ @Override public void collectVectorBatch(AbstractScannedResult scannedResult,
+ CarbonColumnarBatch columnarBatch) {
+ throw new UnsupportedOperationException("Works only for batch collectors");
+ }
+}
[11/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/result/iterator/VectorDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/result/iterator/VectorDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/scan/result/iterator/VectorDetailQueryResultIterator.java
deleted file mode 100644
index 417f597..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/result/iterator/VectorDetailQueryResultIterator.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.result.iterator;
-
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-
-import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.scan.model.QueryModel;
-import org.apache.carbondata.scan.result.vector.CarbonColumnarBatch;
-
-/**
- * It reads the data vector batch format
- */
-public class VectorDetailQueryResultIterator extends AbstractDetailQueryResultIterator<Object> {
-
- private final Object lock = new Object();
-
- public VectorDetailQueryResultIterator(List<BlockExecutionInfo> infos, QueryModel queryModel,
- ExecutorService execService) {
- super(infos, queryModel, execService);
- }
-
- @Override public Object next() {
- throw new UnsupportedOperationException("call processNextBatch instaed");
- }
-
- public void processNextBatch(CarbonColumnarBatch columnarBatch) {
- synchronized (lock) {
- updateDataBlockIterator();
- if (dataBlockIterator != null) {
- dataBlockIterator.processNextBatch(columnarBatch);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/result/vector/CarbonColumnVector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/result/vector/CarbonColumnVector.java b/core/src/main/java/org/apache/carbondata/scan/result/vector/CarbonColumnVector.java
deleted file mode 100644
index 0af1857..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/result/vector/CarbonColumnVector.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.scan.result.vector;
-
-import org.apache.spark.sql.types.Decimal;
-
-public interface CarbonColumnVector {
-
- void putShort(int rowId, short value);
-
- void putInt(int rowId, int value);
-
- void putLong(int rowId, long value);
-
- void putDecimal(int rowId, Decimal value, int precision);
-
- void putDouble(int rowId, double value);
-
- void putBytes(int rowId, byte[] value);
-
- void putBytes(int rowId, int offset, int length, byte[] value);
-
- void putNull(int rowId);
-
- boolean isNull(int rowId);
-
- void putObject(int rowId, Object obj);
-
- Object getData(int rowId);
-
- void reset();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/result/vector/CarbonColumnarBatch.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/result/vector/CarbonColumnarBatch.java b/core/src/main/java/org/apache/carbondata/scan/result/vector/CarbonColumnarBatch.java
deleted file mode 100644
index afc53ca..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/result/vector/CarbonColumnarBatch.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.scan.result.vector;
-
-public class CarbonColumnarBatch {
-
- public CarbonColumnVector[] columnVectors;
-
- private int batchSize;
-
- private int actualSize;
-
- private int rowCounter;
-
- public CarbonColumnarBatch(CarbonColumnVector[] columnVectors, int batchSize) {
- this.columnVectors = columnVectors;
- this.batchSize = batchSize;
- }
-
- public int getBatchSize() {
- return batchSize;
- }
-
- public int getActualSize() {
- return actualSize;
- }
-
- public void setActualSize(int actualSize) {
- this.actualSize = actualSize;
- }
-
- public void reset() {
- actualSize = 0;
- rowCounter = 0;
- for (int i = 0; i < columnVectors.length; i++) {
- columnVectors[i].reset();
- }
- }
-
- public int getRowCounter() {
- return rowCounter;
- }
-
- public void setRowCounter(int rowCounter) {
- this.rowCounter = rowCounter;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/result/vector/ColumnVectorInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/result/vector/ColumnVectorInfo.java b/core/src/main/java/org/apache/carbondata/scan/result/vector/ColumnVectorInfo.java
deleted file mode 100644
index 4b0d7b3..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/result/vector/ColumnVectorInfo.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.result.vector;
-
-import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
-import org.apache.carbondata.scan.filter.GenericQueryType;
-import org.apache.carbondata.scan.model.QueryDimension;
-import org.apache.carbondata.scan.model.QueryMeasure;
-
-public class ColumnVectorInfo implements Comparable<ColumnVectorInfo> {
- public int offset;
- public int size;
- public CarbonColumnVector vector;
- public int vectorOffset;
- public QueryDimension dimension;
- public QueryMeasure measure;
- public int ordinal;
- public DirectDictionaryGenerator directDictionaryGenerator;
- public MeasureDataVectorProcessor.MeasureVectorFiller measureVectorFiller;
- public GenericQueryType genericQueryType;
-
- @Override public int compareTo(ColumnVectorInfo o) {
- return ordinal - o.ordinal;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/result/vector/MeasureDataVectorProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/result/vector/MeasureDataVectorProcessor.java b/core/src/main/java/org/apache/carbondata/scan/result/vector/MeasureDataVectorProcessor.java
deleted file mode 100644
index e713e1b..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/result/vector/MeasureDataVectorProcessor.java
+++ /dev/null
@@ -1,268 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.result.vector;
-
-import java.math.BigDecimal;
-import java.util.BitSet;
-
-import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
-import org.apache.carbondata.core.metadata.DataType;
-
-import org.apache.spark.sql.types.Decimal;
-
-public class MeasureDataVectorProcessor {
-
- public interface MeasureVectorFiller {
-
- void fillMeasureVector(MeasureColumnDataChunk dataChunk, ColumnVectorInfo info);
-
- void fillMeasureVectorForFilter(int[] rowMapping, MeasureColumnDataChunk dataChunk,
- ColumnVectorInfo info);
- }
-
- public static class IntegralMeasureVectorFiller implements MeasureVectorFiller {
-
- @Override
- public void fillMeasureVector(MeasureColumnDataChunk dataChunk, ColumnVectorInfo info) {
- int offset = info.offset;
- int len = offset + info.size;
- int vectorOffset = info.vectorOffset;
- CarbonColumnVector vector = info.vector;
- BitSet nullBitSet = dataChunk.getNullValueIndexHolder().getBitSet();
- for (int i = offset; i < len; i++) {
- if (nullBitSet.get(i)) {
- vector.putNull(vectorOffset);
- } else {
- vector.putInt(vectorOffset,
- (int)dataChunk.getMeasureDataHolder().getReadableLongValueByIndex(i));
- }
- vectorOffset++;
- }
- }
-
- @Override
- public void fillMeasureVectorForFilter(int[] rowMapping, MeasureColumnDataChunk dataChunk,
- ColumnVectorInfo info) {
- int offset = info.offset;
- int len = offset + info.size;
- int vectorOffset = info.vectorOffset;
- CarbonColumnVector vector = info.vector;
- BitSet nullBitSet = dataChunk.getNullValueIndexHolder().getBitSet();
- for (int i = offset; i < len; i++) {
- int currentRow = rowMapping[i];
- if (nullBitSet.get(currentRow)) {
- vector.putNull(vectorOffset);
- } else {
- vector.putInt(vectorOffset,
- (int)dataChunk.getMeasureDataHolder().getReadableLongValueByIndex(currentRow));
- }
- vectorOffset++;
- }
- }
- }
-
- public static class ShortMeasureVectorFiller implements MeasureVectorFiller {
-
- @Override
- public void fillMeasureVector(MeasureColumnDataChunk dataChunk, ColumnVectorInfo info) {
- int offset = info.offset;
- int len = offset + info.size;
- int vectorOffset = info.vectorOffset;
- CarbonColumnVector vector = info.vector;
- BitSet nullBitSet = dataChunk.getNullValueIndexHolder().getBitSet();
- for (int i = offset; i < len; i++) {
- if (nullBitSet.get(i)) {
- vector.putNull(vectorOffset);
- } else {
- vector.putShort(vectorOffset,
- (short) dataChunk.getMeasureDataHolder().getReadableLongValueByIndex(i));
- }
- vectorOffset++;
- }
- }
-
- @Override
- public void fillMeasureVectorForFilter(int[] rowMapping, MeasureColumnDataChunk dataChunk,
- ColumnVectorInfo info) {
- int offset = info.offset;
- int len = offset + info.size;
- int vectorOffset = info.vectorOffset;
- CarbonColumnVector vector = info.vector;
- BitSet nullBitSet = dataChunk.getNullValueIndexHolder().getBitSet();
- for (int i = offset; i < len; i++) {
- int currentRow = rowMapping[i];
- if (nullBitSet.get(currentRow)) {
- vector.putNull(vectorOffset);
- } else {
- vector.putShort(vectorOffset,
- (short) dataChunk.getMeasureDataHolder().getReadableLongValueByIndex(currentRow));
- }
- vectorOffset++;
- }
- }
- }
-
- public static class LongMeasureVectorFiller implements MeasureVectorFiller {
-
- @Override
- public void fillMeasureVector(MeasureColumnDataChunk dataChunk, ColumnVectorInfo info) {
- int offset = info.offset;
- int len = offset + info.size;
- int vectorOffset = info.vectorOffset;
- CarbonColumnVector vector = info.vector;
- BitSet nullBitSet = dataChunk.getNullValueIndexHolder().getBitSet();
- for (int i = offset; i < len; i++) {
- if (nullBitSet.get(i)) {
- vector.putNull(vectorOffset);
- } else {
- vector.putLong(vectorOffset,
- dataChunk.getMeasureDataHolder().getReadableLongValueByIndex(i));
- }
- vectorOffset++;
- }
- }
-
- @Override
- public void fillMeasureVectorForFilter(int[] rowMapping, MeasureColumnDataChunk dataChunk,
- ColumnVectorInfo info) {
- int offset = info.offset;
- int len = offset + info.size;
- int vectorOffset = info.vectorOffset;
- CarbonColumnVector vector = info.vector;
- BitSet nullBitSet = dataChunk.getNullValueIndexHolder().getBitSet();
- for (int i = offset; i < len; i++) {
- int currentRow = rowMapping[i];
- if (nullBitSet.get(currentRow)) {
- vector.putNull(vectorOffset);
- } else {
- vector.putLong(vectorOffset,
- dataChunk.getMeasureDataHolder().getReadableLongValueByIndex(currentRow));
- }
- vectorOffset++;
- }
- }
- }
-
- public static class DecimalMeasureVectorFiller implements MeasureVectorFiller {
-
- @Override
- public void fillMeasureVector(MeasureColumnDataChunk dataChunk, ColumnVectorInfo info) {
- int offset = info.offset;
- int len = offset + info.size;
- int vectorOffset = info.vectorOffset;
- CarbonColumnVector vector = info.vector;
- int precision = info.measure.getMeasure().getPrecision();
- BitSet nullBitSet = dataChunk.getNullValueIndexHolder().getBitSet();
- for (int i = offset; i < len; i++) {
- if (nullBitSet.get(i)) {
- vector.putNull(vectorOffset);
- } else {
- BigDecimal decimal =
- dataChunk.getMeasureDataHolder().getReadableBigDecimalValueByIndex(i);
- Decimal toDecimal = org.apache.spark.sql.types.Decimal.apply(decimal);
- vector.putDecimal(vectorOffset, toDecimal, precision);
- }
- vectorOffset++;
- }
- }
-
- @Override
- public void fillMeasureVectorForFilter(int[] rowMapping, MeasureColumnDataChunk dataChunk,
- ColumnVectorInfo info) {
- int offset = info.offset;
- int len = offset + info.size;
- int vectorOffset = info.vectorOffset;
- CarbonColumnVector vector = info.vector;
- int precision = info.measure.getMeasure().getPrecision();
- BitSet nullBitSet = dataChunk.getNullValueIndexHolder().getBitSet();
- for (int i = offset; i < len; i++) {
- int currentRow = rowMapping[i];
- if (nullBitSet.get(currentRow)) {
- vector.putNull(vectorOffset);
- } else {
- BigDecimal decimal =
- dataChunk.getMeasureDataHolder().getReadableBigDecimalValueByIndex(currentRow);
- Decimal toDecimal = org.apache.spark.sql.types.Decimal.apply(decimal);
- vector.putDecimal(vectorOffset, toDecimal, precision);
- }
- vectorOffset++;
- }
- }
- }
-
- public static class DefaultMeasureVectorFiller implements MeasureVectorFiller {
-
- @Override
- public void fillMeasureVector(MeasureColumnDataChunk dataChunk, ColumnVectorInfo info) {
- int offset = info.offset;
- int len = offset + info.size;
- int vectorOffset = info.vectorOffset;
- CarbonColumnVector vector = info.vector;
- BitSet nullBitSet = dataChunk.getNullValueIndexHolder().getBitSet();
- for (int i = offset; i < len; i++) {
- if (nullBitSet.get(i)) {
- vector.putNull(vectorOffset);
- } else {
- vector.putDouble(vectorOffset,
- dataChunk.getMeasureDataHolder().getReadableDoubleValueByIndex(i));
- }
- vectorOffset++;
- }
- }
-
- @Override
- public void fillMeasureVectorForFilter(int[] rowMapping, MeasureColumnDataChunk dataChunk,
- ColumnVectorInfo info) {
- int offset = info.offset;
- int len = offset + info.size;
- int vectorOffset = info.vectorOffset;
- CarbonColumnVector vector = info.vector;
- BitSet nullBitSet = dataChunk.getNullValueIndexHolder().getBitSet();
- for (int i = offset; i < len; i++) {
- int currentRow = rowMapping[i];
- if (nullBitSet.get(currentRow)) {
- vector.putNull(vectorOffset);
- } else {
- vector.putDouble(vectorOffset,
- dataChunk.getMeasureDataHolder().getReadableDoubleValueByIndex(currentRow));
- }
- vectorOffset++;
- }
- }
- }
-
- public static class MeasureVectorFillerFactory {
-
- public static MeasureVectorFiller getMeasureVectorFiller(DataType dataType) {
- switch (dataType) {
- case SHORT:
- return new ShortMeasureVectorFiller();
- case INT:
- return new IntegralMeasureVectorFiller();
- case LONG:
- return new LongMeasureVectorFiller();
- case DECIMAL:
- return new DecimalMeasureVectorFiller();
- default:
- return new DefaultMeasureVectorFiller();
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/result/vector/impl/CarbonColumnVectorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/result/vector/impl/CarbonColumnVectorImpl.java b/core/src/main/java/org/apache/carbondata/scan/result/vector/impl/CarbonColumnVectorImpl.java
deleted file mode 100644
index 8d0c277..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/result/vector/impl/CarbonColumnVectorImpl.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.result.vector.impl;
-
-import java.util.Arrays;
-import java.util.BitSet;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.scan.result.vector.CarbonColumnVector;
-
-import org.apache.spark.sql.types.Decimal;
-import org.apache.spark.unsafe.types.UTF8String;
-
-public class CarbonColumnVectorImpl implements CarbonColumnVector {
-
- private Object[] data;
-
- private int[] ints;
-
- private long[] longs;
-
- private Decimal[] decimals;
-
- private byte[][] bytes;
-
- private double[] doubles;
-
- private BitSet nullBytes;
-
- private DataType dataType;
-
- public CarbonColumnVectorImpl(int batchSize, DataType dataType) {
- nullBytes = new BitSet(batchSize);
- this.dataType = dataType;
- switch (dataType) {
- case INT:
- ints = new int[batchSize];
- break;
- case LONG:
- longs = new long[batchSize];
- break;
- case DOUBLE:
- doubles = new double[batchSize];
- break;
- case STRING:
- bytes = new byte[batchSize][];
- break;
- case DECIMAL:
- decimals = new Decimal[batchSize];
- break;
- default:
- data = new Object[batchSize];
- }
- }
-
- @Override public void putShort(int rowId, short value) {
-
- }
-
- @Override public void putInt(int rowId, int value) {
- ints[rowId] = value;
- }
-
- @Override public void putLong(int rowId, long value) {
- longs[rowId] = value;
- }
-
- @Override public void putDecimal(int rowId, Decimal value, int precision) {
- decimals[rowId] = value;
- }
-
- @Override public void putDouble(int rowId, double value) {
- doubles[rowId] = value;
- }
-
- @Override public void putBytes(int rowId, byte[] value) {
- bytes[rowId] = value;
- }
-
- @Override public void putBytes(int rowId, int offset, int length, byte[] value) {
-
- }
-
- @Override public void putNull(int rowId) {
- nullBytes.set(rowId);
- }
-
- @Override public boolean isNull(int rowId) {
- return nullBytes.get(rowId);
- }
-
- @Override public void putObject(int rowId, Object obj) {
- data[rowId] = obj;
- }
-
- @Override public Object getData(int rowId) {
- if (nullBytes.get(rowId)) {
- return null;
- }
- switch (dataType) {
- case INT:
- return ints[rowId];
- case LONG:
- return longs[rowId];
- case DOUBLE:
- return doubles[rowId];
- case STRING:
- return UTF8String.fromBytes(bytes[rowId]);
- case DECIMAL:
- return decimals[rowId];
- default:
- return data[rowId];
- }
- }
-
- @Override public void reset() {
- nullBytes.clear();
- switch (dataType) {
- case INT:
- Arrays.fill(ints, 0);
- break;
- case LONG:
- Arrays.fill(longs, 0);
- break;
- case DOUBLE:
- Arrays.fill(doubles, 0);
- break;
- case STRING:
- Arrays.fill(bytes, null);
- break;
- case DECIMAL:
- Arrays.fill(decimals, null);
- break;
- default:
- Arrays.fill(data, null);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/scanner/AbstractBlockletScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/scanner/AbstractBlockletScanner.java b/core/src/main/java/org/apache/carbondata/scan/scanner/AbstractBlockletScanner.java
deleted file mode 100644
index 5343cee..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/scanner/AbstractBlockletScanner.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.scanner;
-
-import java.io.IOException;
-
-import org.apache.carbondata.common.iudprocessor.iuddata.BlockletDeleteDeltaCacheLoader;
-import org.apache.carbondata.common.iudprocessor.iuddata.DeleteDeltaCacheLoaderIntf;
-import org.apache.carbondata.core.stats.QueryStatistic;
-import org.apache.carbondata.core.stats.QueryStatisticsConstants;
-import org.apache.carbondata.core.stats.QueryStatisticsModel;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.processor.BlocksChunkHolder;
-import org.apache.carbondata.scan.result.AbstractScannedResult;
-
-/**
- * Blocklet scanner class to process the block
- */
-public abstract class AbstractBlockletScanner implements BlockletScanner {
-
- /**
- * scanner result
- */
- protected AbstractScannedResult scannedResult;
-
- /**
- * block execution info
- */
- protected BlockExecutionInfo blockExecutionInfo;
-
- public QueryStatisticsModel queryStatisticsModel;
-
- public AbstractBlockletScanner(BlockExecutionInfo tableBlockExecutionInfos) {
- this.blockExecutionInfo = tableBlockExecutionInfos;
- }
-
- @Override public AbstractScannedResult scanBlocklet(BlocksChunkHolder blocksChunkHolder)
- throws IOException, FilterUnsupportedException {
- fillKeyValue(blocksChunkHolder);
- return scannedResult;
- }
-
- protected void fillKeyValue(BlocksChunkHolder blocksChunkHolder) throws IOException {
-
- QueryStatistic totalBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM);
- totalBlockletStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM,
- totalBlockletStatistic.getCount() + 1);
- queryStatisticsModel.getRecorder().recordStatistics(totalBlockletStatistic);
- QueryStatistic validScannedBlockletStatistic = queryStatisticsModel
- .getStatisticsTypeAndObjMap().get(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM);
- validScannedBlockletStatistic
- .addCountStatistic(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM,
- validScannedBlockletStatistic.getCount() + 1);
- queryStatisticsModel.getRecorder().recordStatistics(validScannedBlockletStatistic);
- scannedResult.reset();
- scannedResult.setNumberOfRows(blocksChunkHolder.getDataBlock().nodeSize());
- scannedResult.setBlockletId(
- blockExecutionInfo.getBlockId() + CarbonCommonConstants.FILE_SEPARATOR
- + blocksChunkHolder.getDataBlock().nodeNumber());
- scannedResult.setDimensionChunks(blocksChunkHolder.getDataBlock()
- .getDimensionChunks(blocksChunkHolder.getFileReader(),
- blockExecutionInfo.getAllSelectedDimensionBlocksIndexes()));
- scannedResult.setMeasureChunks(blocksChunkHolder.getDataBlock()
- .getMeasureChunks(blocksChunkHolder.getFileReader(),
- blockExecutionInfo.getAllSelectedMeasureBlocksIndexes()));
- // loading delete data cache in blockexecutioninfo instance
- DeleteDeltaCacheLoaderIntf deleteCacheLoader =
- new BlockletDeleteDeltaCacheLoader(scannedResult.getBlockletId(),
- blocksChunkHolder.getDataBlock(), blockExecutionInfo.getAbsoluteTableIdentifier());
- deleteCacheLoader.loadDeleteDeltaFileDataToCache();
- scannedResult
- .setBlockletDeleteDeltaCache(blocksChunkHolder.getDataBlock().getDeleteDeltaDataCache());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/scanner/BlockletScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/scanner/BlockletScanner.java b/core/src/main/java/org/apache/carbondata/scan/scanner/BlockletScanner.java
deleted file mode 100644
index 39a20b2..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/scanner/BlockletScanner.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.scanner;
-
-import java.io.IOException;
-
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.processor.BlocksChunkHolder;
-import org.apache.carbondata.scan.result.AbstractScannedResult;
-
-/**
- * Interface for processing the block
- * Processing can be filter based processing or non filter based processing
- */
-public interface BlockletScanner {
-
- /**
- * Below method will used to process the block data and get the scanned result
- *
- * @param blocksChunkHolder block chunk which holds the block data
- * @return scannerResult
- * result after processing
- */
- AbstractScannedResult scanBlocklet(BlocksChunkHolder blocksChunkHolder)
- throws IOException, FilterUnsupportedException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/scanner/impl/FilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/scanner/impl/FilterScanner.java b/core/src/main/java/org/apache/carbondata/scan/scanner/impl/FilterScanner.java
deleted file mode 100644
index e947e62..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/scanner/impl/FilterScanner.java
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.scanner.impl;
-
-import java.io.IOException;
-import java.util.BitSet;
-
-import org.apache.carbondata.common.iudprocessor.iuddata.BlockletDeleteDeltaCacheLoader;
-import org.apache.carbondata.common.iudprocessor.iuddata.DeleteDeltaCacheLoaderIntf;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
-import org.apache.carbondata.core.stats.QueryStatistic;
-import org.apache.carbondata.core.stats.QueryStatisticsConstants;
-import org.apache.carbondata.core.stats.QueryStatisticsModel;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.executer.FilterExecuter;
-import org.apache.carbondata.scan.processor.BlocksChunkHolder;
-import org.apache.carbondata.scan.result.AbstractScannedResult;
-import org.apache.carbondata.scan.result.impl.FilterQueryScannedResult;
-import org.apache.carbondata.scan.scanner.AbstractBlockletScanner;
-
-/**
- * Below class will be used for filter query processing
- * this class will be first apply the filter then it will read the block if
- * required and return the scanned result
- */
-public class FilterScanner extends AbstractBlockletScanner {
-
- /**
- * filter tree
- */
- private FilterExecuter filterExecuter;
- /**
- * this will be used to apply min max
- * this will be useful for dimension column which is on the right side
- * as node finder will always give tentative blocks, if column data stored individually
- * and data is in sorted order then we can check whether filter is in the range of min max or not
- * if it present then only we can apply filter on complete data.
- * this will be very useful in case of sparse data when rows are
- * repeating.
- */
- private boolean isMinMaxEnabled;
-
- private QueryStatisticsModel queryStatisticsModel;
-
- public FilterScanner(BlockExecutionInfo blockExecutionInfo,
- QueryStatisticsModel queryStatisticsModel) {
- super(blockExecutionInfo);
- scannedResult = new FilterQueryScannedResult(blockExecutionInfo);
- // to check whether min max is enabled or not
- String minMaxEnableValue = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.CARBON_QUERY_MIN_MAX_ENABLED,
- CarbonCommonConstants.MIN_MAX_DEFAULT_VALUE);
- if (null != minMaxEnableValue) {
- isMinMaxEnabled = Boolean.parseBoolean(minMaxEnableValue);
- }
- // get the filter tree
- this.filterExecuter = blockExecutionInfo.getFilterExecuterTree();
- this.queryStatisticsModel = queryStatisticsModel;
- }
-
- /**
- * Below method will be used to process the block
- *
- * @param blocksChunkHolder block chunk holder which holds the data
- * @throws FilterUnsupportedException
- */
- @Override public AbstractScannedResult scanBlocklet(BlocksChunkHolder blocksChunkHolder)
- throws IOException, FilterUnsupportedException {
- fillScannedResult(blocksChunkHolder);
- return scannedResult;
- }
-
- /**
- * This method will process the data in below order
- * 1. first apply min max on the filter tree and check whether any of the filter
- * is fall on the range of min max, if not then return empty result
- * 2. If filter falls on min max range then apply filter on actual
- * data and get the filtered row index
- * 3. if row index is empty then return the empty result
- * 4. if row indexes is not empty then read only those blocks(measure or dimension)
- * which was present in the query but not present in the filter, as while applying filter
- * some of the blocks where already read and present in chunk holder so not need to
- * read those blocks again, this is to avoid reading of same blocks which was already read
- * 5. Set the blocks and filter indexes to result
- *
- * @param blocksChunkHolder
- * @throws FilterUnsupportedException
- */
- private void fillScannedResult(BlocksChunkHolder blocksChunkHolder)
- throws FilterUnsupportedException, IOException {
- scannedResult.reset();
- scannedResult.setBlockletId(
- blockExecutionInfo.getBlockId() + CarbonCommonConstants.FILE_SEPARATOR + blocksChunkHolder
- .getDataBlock().nodeNumber());
- // apply min max
- if (isMinMaxEnabled) {
- BitSet bitSet = this.filterExecuter
- .isScanRequired(blocksChunkHolder.getDataBlock().getColumnsMaxValue(),
- blocksChunkHolder.getDataBlock().getColumnsMinValue());
- if (bitSet.isEmpty()) {
- scannedResult.setNumberOfRows(0);
- scannedResult.setIndexes(new int[0]);
- CarbonUtil.freeMemory(blocksChunkHolder.getDimensionDataChunk(),
- blocksChunkHolder.getMeasureDataChunk());
- return;
- }
- }
- // apply filter on actual data
- BitSet bitSet = this.filterExecuter.applyFilter(blocksChunkHolder);
- // if indexes is empty then return with empty result
- if (bitSet.isEmpty()) {
- scannedResult.setNumberOfRows(0);
- scannedResult.setIndexes(new int[0]);
- return;
- }
- // valid scanned blocklet
- QueryStatistic validScannedBlockletStatistic = queryStatisticsModel.getStatisticsTypeAndObjMap()
- .get(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM);
- validScannedBlockletStatistic
- .addCountStatistic(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM,
- validScannedBlockletStatistic.getCount() + 1);
- queryStatisticsModel.getRecorder().recordStatistics(validScannedBlockletStatistic);
- // get the row indexes from bot set
- int[] indexes = new int[bitSet.cardinality()];
- int index = 0;
- for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) {
- indexes[index++] = i;
- }
- // loading delete data cache in blockexecutioninfo instance
- DeleteDeltaCacheLoaderIntf deleteCacheLoader =
- new BlockletDeleteDeltaCacheLoader(scannedResult.getBlockletId(),
- blocksChunkHolder.getDataBlock(), blockExecutionInfo.getAbsoluteTableIdentifier());
- deleteCacheLoader.loadDeleteDeltaFileDataToCache();
- scannedResult
- .setBlockletDeleteDeltaCache(blocksChunkHolder.getDataBlock().getDeleteDeltaDataCache());
- FileHolder fileReader = blocksChunkHolder.getFileReader();
- int[][] allSelectedDimensionBlocksIndexes =
- blockExecutionInfo.getAllSelectedDimensionBlocksIndexes();
- DimensionColumnDataChunk[] projectionListDimensionChunk = blocksChunkHolder.getDataBlock()
- .getDimensionChunks(fileReader, allSelectedDimensionBlocksIndexes);
-
- DimensionColumnDataChunk[] dimensionColumnDataChunk =
- new DimensionColumnDataChunk[blockExecutionInfo.getTotalNumberDimensionBlock()];
- // read dimension chunk blocks from file which is not present
- for (int i = 0; i < dimensionColumnDataChunk.length; i++) {
- if (null != blocksChunkHolder.getDimensionDataChunk()[i]) {
- dimensionColumnDataChunk[i] = blocksChunkHolder.getDimensionDataChunk()[i];
- }
- }
- for (int i = 0; i < allSelectedDimensionBlocksIndexes.length; i++) {
- System.arraycopy(projectionListDimensionChunk, allSelectedDimensionBlocksIndexes[i][0],
- dimensionColumnDataChunk, allSelectedDimensionBlocksIndexes[i][0],
- allSelectedDimensionBlocksIndexes[i][1] + 1 - allSelectedDimensionBlocksIndexes[i][0]);
- }
- MeasureColumnDataChunk[] measureColumnDataChunk =
- new MeasureColumnDataChunk[blockExecutionInfo.getTotalNumberOfMeasureBlock()];
- int[][] allSelectedMeasureBlocksIndexes =
- blockExecutionInfo.getAllSelectedMeasureBlocksIndexes();
- MeasureColumnDataChunk[] projectionListMeasureChunk = blocksChunkHolder.getDataBlock()
- .getMeasureChunks(fileReader, allSelectedMeasureBlocksIndexes);
- // read the measure chunk blocks which is not present
- for (int i = 0; i < measureColumnDataChunk.length; i++) {
- if (null != blocksChunkHolder.getMeasureDataChunk()[i]) {
- measureColumnDataChunk[i] = blocksChunkHolder.getMeasureDataChunk()[i];
- }
- }
- for (int i = 0; i < allSelectedMeasureBlocksIndexes.length; i++) {
- System.arraycopy(projectionListMeasureChunk, allSelectedMeasureBlocksIndexes[i][0],
- measureColumnDataChunk, allSelectedMeasureBlocksIndexes[i][0],
- allSelectedMeasureBlocksIndexes[i][1] + 1 - allSelectedMeasureBlocksIndexes[i][0]);
- }
- scannedResult.setDimensionChunks(dimensionColumnDataChunk);
- scannedResult.setIndexes(indexes);
- scannedResult.setMeasureChunks(measureColumnDataChunk);
- scannedResult.setNumberOfRows(indexes.length);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/scanner/impl/NonFilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/scanner/impl/NonFilterScanner.java b/core/src/main/java/org/apache/carbondata/scan/scanner/impl/NonFilterScanner.java
deleted file mode 100644
index 0cea193..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/scanner/impl/NonFilterScanner.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.scanner.impl;
-
-import org.apache.carbondata.core.stats.QueryStatisticsModel;
-import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.scan.result.impl.NonFilterQueryScannedResult;
-import org.apache.carbondata.scan.scanner.AbstractBlockletScanner;
-
-/**
- * Non filter processor which will be used for non filter query
- * In case of non filter query we just need to read all the blocks requested in the
- * query and pass it to scanned result
- */
-public class NonFilterScanner extends AbstractBlockletScanner {
-
- public NonFilterScanner(BlockExecutionInfo blockExecutionInfo,
- QueryStatisticsModel queryStatisticsModel) {
- super(blockExecutionInfo);
- // as its a non filter query creating a non filter query scanned result object
- scannedResult = new NonFilterQueryScannedResult(blockExecutionInfo);
- super.queryStatisticsModel = queryStatisticsModel;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/wrappers/ByteArrayWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/wrappers/ByteArrayWrapper.java b/core/src/main/java/org/apache/carbondata/scan/wrappers/ByteArrayWrapper.java
deleted file mode 100644
index 1805636..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/wrappers/ByteArrayWrapper.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.wrappers;
-
-import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
-
-/**
- * This class will store the dimension column data when query is executed
- * This can be used as a key for aggregation
- */
-public class ByteArrayWrapper implements Comparable<ByteArrayWrapper> {
-
- /**
- * to store key which is generated using
- * key generator
- */
- protected byte[] dictionaryKey;
-
- /**
- * to store no dictionary column data
- */
- protected byte[][] complexTypesKeys;
-
- /**
- * to store no dictionary column data
- */
- protected byte[][] noDictionaryKeys;
-
- /**
- * contains value of implicit columns in byte array format
- */
- protected byte[] implicitColumnByteArray;
-
- public ByteArrayWrapper() {
- }
-
- /**
- * @return the dictionaryKey
- */
- public byte[] getDictionaryKey() {
- return dictionaryKey;
- }
-
- /**
- * @param dictionaryKey the dictionaryKey to set
- */
- public void setDictionaryKey(byte[] dictionaryKey) {
- this.dictionaryKey = dictionaryKey;
- }
-
- /**
- * @param noDictionaryKeys the noDictionaryKeys to set
- */
- public void setNoDictionaryKeys(byte[][] noDictionaryKeys) {
- this.noDictionaryKeys = noDictionaryKeys;
- }
-
- /**
- * to get the no dictionary column data
- *
- * @param index of the no dictionary key
- * @return no dictionary key for the index
- */
- public byte[] getNoDictionaryKeyByIndex(int index) {
- return this.noDictionaryKeys[index];
- }
-
- /**
- * to get the no dictionary column data
- *
- * @param index of the no dictionary key
- * @return no dictionary key for the index
- */
- public byte[] getComplexTypeByIndex(int index) {
- return this.complexTypesKeys[index];
- }
-
- /**
- * to generate the hash code
- */
- @Override public int hashCode() {
- // first generate the has code of the dictionary column
- int len = dictionaryKey.length;
- int result = 1;
- for (int j = 0; j < len; j++) {
- result = 31 * result + dictionaryKey[j];
- }
- // then no dictionary column
- for (byte[] directSurrogateValue : noDictionaryKeys) {
- for (int i = 0; i < directSurrogateValue.length; i++) {
- result = 31 * result + directSurrogateValue[i];
- }
- }
- // then for complex type
- for (byte[] complexTypeKey : complexTypesKeys) {
- for (int i = 0; i < complexTypeKey.length; i++) {
- result = 31 * result + complexTypeKey[i];
- }
- }
- return result;
- }
-
- /**
- * to validate the two
- *
- * @param other object
- */
- @Override public boolean equals(Object other) {
- if (null == other || !(other instanceof ByteArrayWrapper)) {
- return false;
- }
- boolean result = false;
- // Comparison will be as follows
- // first compare the no dictionary column
- // if it is not equal then return false
- // if it is equal then compare the complex column
- // if it is also equal then compare dictionary column
- byte[][] noDictionaryKeysOther = ((ByteArrayWrapper) other).noDictionaryKeys;
- if (noDictionaryKeysOther.length != noDictionaryKeys.length) {
- return false;
- } else {
- for (int i = 0; i < noDictionaryKeys.length; i++) {
- result = UnsafeComparer.INSTANCE.equals(noDictionaryKeys[i], noDictionaryKeysOther[i]);
- if (!result) {
- return false;
- }
- }
- }
-
- byte[][] complexTypesKeysOther = ((ByteArrayWrapper) other).complexTypesKeys;
- if (complexTypesKeysOther.length != complexTypesKeys.length) {
- return false;
- } else {
- for (int i = 0; i < complexTypesKeys.length; i++) {
- result = UnsafeComparer.INSTANCE.equals(complexTypesKeys[i], complexTypesKeysOther[i]);
- if (!result) {
- return false;
- }
- }
- }
-
- return UnsafeComparer.INSTANCE.equals(dictionaryKey, ((ByteArrayWrapper) other).dictionaryKey);
- }
-
- /**
- * Compare method for ByteArrayWrapper class this will used to compare Two
- * ByteArrayWrapper data object, basically it will compare two byte array
- *
- * @param other ArrayWrapper Object
- */
- @Override public int compareTo(ByteArrayWrapper other) {
- // compare will be as follows
- //compare dictionary column
- // then no dictionary column
- // then complex type column data
- int compareTo = UnsafeComparer.INSTANCE.compareTo(dictionaryKey, other.dictionaryKey);
- if (compareTo == 0) {
- for (int i = 0; i < noDictionaryKeys.length; i++) {
- compareTo =
- UnsafeComparer.INSTANCE.compareTo(noDictionaryKeys[i], other.noDictionaryKeys[i]);
- if (compareTo != 0) {
- return compareTo;
- }
- }
- }
- if (compareTo == 0) {
- for (int i = 0; i < complexTypesKeys.length; i++) {
- compareTo =
- UnsafeComparer.INSTANCE.compareTo(complexTypesKeys[i], other.complexTypesKeys[i]);
- if (compareTo != 0) {
- return compareTo;
- }
- }
- }
- return compareTo;
- }
-
- /**
- * @return the complexTypesKeys
- */
- public byte[][] getComplexTypesKeys() {
- return complexTypesKeys;
- }
-
- /**
- * @param complexTypesKeys the complexTypesKeys to set
- */
- public void setComplexTypesKeys(byte[][] complexTypesKeys) {
- this.complexTypesKeys = complexTypesKeys;
- }
-
- /**
- * @return
- */
- public byte[] getImplicitColumnByteArray() {
- return implicitColumnByteArray;
- }
-
- /**
- * @param implicitColumnByteArray
- */
- public void setImplicitColumnByteArray(byte[] implicitColumnByteArray) {
- this.implicitColumnByteArray = implicitColumnByteArray;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/AbsoluteTableIdentifierTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/AbsoluteTableIdentifierTest.java b/core/src/test/java/org/apache/carbondata/core/AbsoluteTableIdentifierTest.java
new file mode 100644
index 0000000..aaf0d97
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/AbsoluteTableIdentifierTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core;
+
+import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.CarbonTableIdentifier;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertEquals;
+
+public class AbsoluteTableIdentifierTest {
+ static AbsoluteTableIdentifier absoluteTableIdentifier;
+ static AbsoluteTableIdentifier absoluteTableIdentifier1;
+ static AbsoluteTableIdentifier absoluteTableIdentifier2;
+ static AbsoluteTableIdentifier absoluteTableIdentifier3;
+ static AbsoluteTableIdentifier absoluteTableIdentifier4;
+
+ @BeforeClass public static void setup() {
+ absoluteTableIdentifier = new AbsoluteTableIdentifier("storePath",
+ new CarbonTableIdentifier("databaseName", "tableName", "tableId"));
+ absoluteTableIdentifier1 = new AbsoluteTableIdentifier("dummy", null);
+ absoluteTableIdentifier2 = new AbsoluteTableIdentifier("dumgfhmy", null);
+ absoluteTableIdentifier3 =
+ new AbsoluteTableIdentifier("duhgmmy", new CarbonTableIdentifier("dummy", "dumy", "dmy"));
+ absoluteTableIdentifier4 = new AbsoluteTableIdentifier("storePath",
+ new CarbonTableIdentifier("databaseName", "tableName", "tableId"));
+ }
+
+ @Test public void equalsTestWithSameInstance() {
+ Boolean res = absoluteTableIdentifier.equals("wrong data");
+ assert (!res);
+ }
+
+ @Test public void equalsTestWithNullObject() {
+ Boolean res = absoluteTableIdentifier.equals(null);
+ assert (!res);
+ }
+
+ @Test public void equalsTestWithotherObject() {
+ Boolean res = absoluteTableIdentifier1.equals(absoluteTableIdentifier);
+ assert (!res);
+ }
+
+ @Test public void equalsTestWithSameObj() {
+ Boolean res = absoluteTableIdentifier.equals(absoluteTableIdentifier);
+ assert (res);
+ }
+
+ @Test public void equalsTestWithNullColumnIdentifier() {
+ Boolean res = absoluteTableIdentifier1.equals(absoluteTableIdentifier2);
+ assert (!res);
+ }
+
+ @Test public void equalsTestWithEqualColumnIdentifier() {
+ Boolean res = absoluteTableIdentifier3.equals(absoluteTableIdentifier4);
+ assert (!res);
+ }
+
+ @Test public void equalsTestWithEqualAbsoluteTableIdentifier() {
+ Boolean res = absoluteTableIdentifier.equals(absoluteTableIdentifier4);
+ assert (res);
+ }
+
+ @Test public void hashCodeTest() {
+ int res = absoluteTableIdentifier4.hashCode();
+ int expectedResult = 804398706;
+ assertEquals(res, expectedResult);
+ }
+
+ @Test public void gettablePathTest() {
+ String res = absoluteTableIdentifier4.getTablePath();
+ assert (res.equals("storePath/databaseName/tableName"));
+ }
+
+ @Test public void fromTablePathTest() {
+ AbsoluteTableIdentifier absoluteTableIdentifierTest =
+ AbsoluteTableIdentifier.fromTablePath("storePath/databaseName/tableName");
+ assert (absoluteTableIdentifierTest.getStorePath()
+ .equals(absoluteTableIdentifier4.getStorePath()));
+ }
+
+ @Test(expected = IllegalArgumentException.class) public void fromTablePathWithExceptionTest() {
+ AbsoluteTableIdentifier absoluteTableIdentifierTest =
+ AbsoluteTableIdentifier.fromTablePath("storePath/databaseName");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/CarbonTableIdentifierTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/CarbonTableIdentifierTest.java b/core/src/test/java/org/apache/carbondata/core/CarbonTableIdentifierTest.java
new file mode 100644
index 0000000..942291c
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/CarbonTableIdentifierTest.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core;
+
+import org.apache.carbondata.core.CarbonTableIdentifier;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class CarbonTableIdentifierTest {
+
+ static CarbonTableIdentifier carbonTableIdentifier;
+ static CarbonTableIdentifier carbonTableIdentifier2;
+
+ @BeforeClass public static void setup() {
+ carbonTableIdentifier = new CarbonTableIdentifier("DatabseName", "tableName", "tableId");
+
+ }
+
+ @Test public void equalsTestWithSameObject() {
+ Boolean res = carbonTableIdentifier.equals(carbonTableIdentifier);
+ assert (res);
+ }
+
+ @Test public void equalsTestWithSimilarObject() {
+ CarbonTableIdentifier carbonTableIdentifierTest =
+ new CarbonTableIdentifier("DatabseName", "tableName", "tableId");
+ Boolean res = carbonTableIdentifier.equals(carbonTableIdentifierTest);
+ assert (res);
+ }
+
+ @Test public void equalsTestWithNullrObject() {
+ Boolean res = carbonTableIdentifier.equals(carbonTableIdentifier2);
+ assert (!res);
+ }
+
+ @Test public void equalsTestWithStringrObject() {
+ Boolean res = carbonTableIdentifier.equals("different class object");
+ assert (!res);
+ }
+
+ @Test public void equalsTestWithoutDatabaseName() {
+ CarbonTableIdentifier carbonTableIdentifierTest =
+ new CarbonTableIdentifier(null, "tableName", "tableId");
+ Boolean res = carbonTableIdentifierTest.equals(carbonTableIdentifier);
+ assert (!res);
+ }
+
+ @Test public void equalsTestWithoutTableId() {
+ CarbonTableIdentifier carbonTableIdentifierTest =
+ new CarbonTableIdentifier("DatabseName", "tableName", null);
+ Boolean res = carbonTableIdentifierTest.equals(carbonTableIdentifier);
+ assert (!res);
+ }
+
+ @Test public void equalsTestWithDifferentTableId() {
+ CarbonTableIdentifier carbonTableIdentifierTest =
+ new CarbonTableIdentifier("DatabseName", "tableName", "diffTableId");
+ Boolean res = carbonTableIdentifierTest.equals(carbonTableIdentifier);
+ assert (!res);
+ }
+
+ @Test public void equalsTestWithNullTableName() {
+ CarbonTableIdentifier carbonTableIdentifierTest =
+ new CarbonTableIdentifier("DatabseName", null, "tableId");
+ Boolean res = carbonTableIdentifierTest.equals(carbonTableIdentifier);
+ assert (!res);
+ }
+
+ @Test public void equalsTestWithDifferentTableName() {
+ CarbonTableIdentifier carbonTableIdentifierTest =
+ new CarbonTableIdentifier("DatabseName", "diffTableName", "tableId");
+ Boolean res = carbonTableIdentifierTest.equals(carbonTableIdentifier);
+ assert (!res);
+ }
+
+ @Test public void toStringTest() {
+ String res = carbonTableIdentifier.toString();
+ System.out.printf("sfdsdf " + res);
+ assert (res.equals("DatabseName_tableName"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/ColumnIdentifierTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/ColumnIdentifierTest.java b/core/src/test/java/org/apache/carbondata/core/ColumnIdentifierTest.java
new file mode 100644
index 0000000..f4e7094
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/ColumnIdentifierTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core;
+
+import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.DataType;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static junit.framework.TestCase.assertEquals;
+
+public class ColumnIdentifierTest {
+
+ static ColumnIdentifier columnIdentifier;
+ static Map<String, String> columnProperties;
+
+ @BeforeClass public static void setup() {
+ columnProperties = new HashMap<String, String>();
+ columnProperties.put("key", "value");
+ columnIdentifier = new ColumnIdentifier("columnId", columnProperties, DataType.INT);
+ }
+
+ @Test public void hashCodeTest() {
+ int res = columnIdentifier.hashCode();
+ int expectedResult = -623419600;
+ assertEquals(res, expectedResult);
+ }
+
+ @Test public void equalsTestwithSameObject() {
+ Boolean res = columnIdentifier.equals(columnIdentifier);
+ assert (res);
+ }
+
+ @Test public void equalsTestwithSimilarObject() {
+ ColumnIdentifier columnIdentifierTest =
+ new ColumnIdentifier("columnId", columnProperties, DataType.INT);
+ Boolean res = columnIdentifier.equals(columnIdentifierTest);
+ assert (res);
+ }
+
+ @Test public void equalsTestwithNullObject() {
+ Boolean res = columnIdentifier.equals(null);
+ assert (!res);
+ }
+
+ @Test public void equalsTestwithStringObject() {
+ Boolean res = columnIdentifier.equals("String Object");
+ assert (!res);
+ }
+
+ @Test public void equalsTestwithNullColumnId() {
+ ColumnIdentifier columnIdentifierTest =
+ new ColumnIdentifier(null, columnProperties, DataType.INT);
+ Boolean res = columnIdentifierTest.equals(columnIdentifier);
+ assert (!res);
+ }
+
+ @Test public void equalsTestwithDiffColumnId() {
+ ColumnIdentifier columnIdentifierTest =
+ new ColumnIdentifier("diffColumnId", columnProperties, DataType.INT);
+ Boolean res = columnIdentifierTest.equals(columnIdentifier);
+ assert (!res);
+ }
+
+ @Test public void toStringTest() {
+ String res = columnIdentifier.toString();
+ assert (res.equals("ColumnIdentifier [columnId=columnId]"));
+ }
+
+ @Test public void getColumnPropertyTest() {
+ ColumnIdentifier columnIdentifierTest =
+ new ColumnIdentifier("diffColumnId", null, DataType.INT);
+ String res = columnIdentifierTest.getColumnProperty("key");
+ assertEquals(res, null);
+ }
+
+ @Test public void getColumnPropertyTestwithNull() {
+ assert (columnIdentifier.getColumnProperty("key").equals("value"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
index 904ddcb..4fd4d37 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
@@ -37,8 +37,8 @@ import org.apache.carbondata.core.ColumnIdentifier;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.writer.CarbonDictionaryWriter;
import org.apache.carbondata.core.writer.CarbonDictionaryWriterImpl;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/carbon/AbsoluteTableIdentifierTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/AbsoluteTableIdentifierTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/AbsoluteTableIdentifierTest.java
deleted file mode 100644
index ef31d06..0000000
--- a/core/src/test/java/org/apache/carbondata/core/carbon/AbsoluteTableIdentifierTest.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.carbon;
-
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static junit.framework.TestCase.assertEquals;
-
-public class AbsoluteTableIdentifierTest {
- static AbsoluteTableIdentifier absoluteTableIdentifier;
- static AbsoluteTableIdentifier absoluteTableIdentifier1;
- static AbsoluteTableIdentifier absoluteTableIdentifier2;
- static AbsoluteTableIdentifier absoluteTableIdentifier3;
- static AbsoluteTableIdentifier absoluteTableIdentifier4;
-
- @BeforeClass public static void setup() {
- absoluteTableIdentifier = new AbsoluteTableIdentifier("storePath",
- new CarbonTableIdentifier("databaseName", "tableName", "tableId"));
- absoluteTableIdentifier1 = new AbsoluteTableIdentifier("dummy", null);
- absoluteTableIdentifier2 = new AbsoluteTableIdentifier("dumgfhmy", null);
- absoluteTableIdentifier3 =
- new AbsoluteTableIdentifier("duhgmmy", new CarbonTableIdentifier("dummy", "dumy", "dmy"));
- absoluteTableIdentifier4 = new AbsoluteTableIdentifier("storePath",
- new CarbonTableIdentifier("databaseName", "tableName", "tableId"));
- }
-
- @Test public void equalsTestWithSameInstance() {
- Boolean res = absoluteTableIdentifier.equals("wrong data");
- assert (!res);
- }
-
- @Test public void equalsTestWithNullObject() {
- Boolean res = absoluteTableIdentifier.equals(null);
- assert (!res);
- }
-
- @Test public void equalsTestWithotherObject() {
- Boolean res = absoluteTableIdentifier1.equals(absoluteTableIdentifier);
- assert (!res);
- }
-
- @Test public void equalsTestWithSameObj() {
- Boolean res = absoluteTableIdentifier.equals(absoluteTableIdentifier);
- assert (res);
- }
-
- @Test public void equalsTestWithNullColumnIdentifier() {
- Boolean res = absoluteTableIdentifier1.equals(absoluteTableIdentifier2);
- assert (!res);
- }
-
- @Test public void equalsTestWithEqualColumnIdentifier() {
- Boolean res = absoluteTableIdentifier3.equals(absoluteTableIdentifier4);
- assert (!res);
- }
-
- @Test public void equalsTestWithEqualAbsoluteTableIdentifier() {
- Boolean res = absoluteTableIdentifier.equals(absoluteTableIdentifier4);
- assert (res);
- }
-
- @Test public void hashCodeTest() {
- int res = absoluteTableIdentifier4.hashCode();
- int expectedResult = 804398706;
- assertEquals(res, expectedResult);
- }
-
- @Test public void gettablePathTest() {
- String res = absoluteTableIdentifier4.getTablePath();
- assert (res.equals("storePath/databaseName/tableName"));
- }
-
- @Test public void fromTablePathTest() {
- AbsoluteTableIdentifier absoluteTableIdentifierTest =
- AbsoluteTableIdentifier.fromTablePath("storePath/databaseName/tableName");
- assert (absoluteTableIdentifierTest.getStorePath()
- .equals(absoluteTableIdentifier4.getStorePath()));
- }
-
- @Test(expected = IllegalArgumentException.class) public void fromTablePathWithExceptionTest() {
- AbsoluteTableIdentifier absoluteTableIdentifierTest =
- AbsoluteTableIdentifier.fromTablePath("storePath/databaseName");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/carbon/CarbonTableIdentifierTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/CarbonTableIdentifierTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/CarbonTableIdentifierTest.java
deleted file mode 100644
index 1156d35..0000000
--- a/core/src/test/java/org/apache/carbondata/core/carbon/CarbonTableIdentifierTest.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.carbon;
-
-import org.apache.carbondata.core.CarbonTableIdentifier;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class CarbonTableIdentifierTest {
-
- static CarbonTableIdentifier carbonTableIdentifier;
- static CarbonTableIdentifier carbonTableIdentifier2;
-
- @BeforeClass public static void setup() {
- carbonTableIdentifier = new CarbonTableIdentifier("DatabseName", "tableName", "tableId");
-
- }
-
- @Test public void equalsTestWithSameObject() {
- Boolean res = carbonTableIdentifier.equals(carbonTableIdentifier);
- assert (res);
- }
-
- @Test public void equalsTestWithSimilarObject() {
- CarbonTableIdentifier carbonTableIdentifierTest =
- new CarbonTableIdentifier("DatabseName", "tableName", "tableId");
- Boolean res = carbonTableIdentifier.equals(carbonTableIdentifierTest);
- assert (res);
- }
-
- @Test public void equalsTestWithNullrObject() {
- Boolean res = carbonTableIdentifier.equals(carbonTableIdentifier2);
- assert (!res);
- }
-
- @Test public void equalsTestWithStringrObject() {
- Boolean res = carbonTableIdentifier.equals("different class object");
- assert (!res);
- }
-
- @Test public void equalsTestWithoutDatabaseName() {
- CarbonTableIdentifier carbonTableIdentifierTest =
- new CarbonTableIdentifier(null, "tableName", "tableId");
- Boolean res = carbonTableIdentifierTest.equals(carbonTableIdentifier);
- assert (!res);
- }
-
- @Test public void equalsTestWithoutTableId() {
- CarbonTableIdentifier carbonTableIdentifierTest =
- new CarbonTableIdentifier("DatabseName", "tableName", null);
- Boolean res = carbonTableIdentifierTest.equals(carbonTableIdentifier);
- assert (!res);
- }
-
- @Test public void equalsTestWithDifferentTableId() {
- CarbonTableIdentifier carbonTableIdentifierTest =
- new CarbonTableIdentifier("DatabseName", "tableName", "diffTableId");
- Boolean res = carbonTableIdentifierTest.equals(carbonTableIdentifier);
- assert (!res);
- }
-
- @Test public void equalsTestWithNullTableName() {
- CarbonTableIdentifier carbonTableIdentifierTest =
- new CarbonTableIdentifier("DatabseName", null, "tableId");
- Boolean res = carbonTableIdentifierTest.equals(carbonTableIdentifier);
- assert (!res);
- }
-
- @Test public void equalsTestWithDifferentTableName() {
- CarbonTableIdentifier carbonTableIdentifierTest =
- new CarbonTableIdentifier("DatabseName", "diffTableName", "tableId");
- Boolean res = carbonTableIdentifierTest.equals(carbonTableIdentifier);
- assert (!res);
- }
-
- @Test public void toStringTest() {
- String res = carbonTableIdentifier.toString();
- System.out.printf("sfdsdf " + res);
- assert (res.equals("DatabseName_tableName"));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/carbon/ColumnIdentifierTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/ColumnIdentifierTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/ColumnIdentifierTest.java
deleted file mode 100644
index 4a5fcfd..0000000
--- a/core/src/test/java/org/apache/carbondata/core/carbon/ColumnIdentifierTest.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.carbon;
-
-import org.apache.carbondata.core.ColumnIdentifier;
-import org.apache.carbondata.core.metadata.DataType;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import static junit.framework.TestCase.assertEquals;
-
-public class ColumnIdentifierTest {
-
- static ColumnIdentifier columnIdentifier;
- static Map<String, String> columnProperties;
-
- @BeforeClass public static void setup() {
- columnProperties = new HashMap<String, String>();
- columnProperties.put("key", "value");
- columnIdentifier = new ColumnIdentifier("columnId", columnProperties, DataType.INT);
- }
-
- @Test public void hashCodeTest() {
- int res = columnIdentifier.hashCode();
- int expectedResult = -623419600;
- assertEquals(res, expectedResult);
- }
-
- @Test public void equalsTestwithSameObject() {
- Boolean res = columnIdentifier.equals(columnIdentifier);
- assert (res);
- }
-
- @Test public void equalsTestwithSimilarObject() {
- ColumnIdentifier columnIdentifierTest =
- new ColumnIdentifier("columnId", columnProperties, DataType.INT);
- Boolean res = columnIdentifier.equals(columnIdentifierTest);
- assert (res);
- }
-
- @Test public void equalsTestwithNullObject() {
- Boolean res = columnIdentifier.equals(null);
- assert (!res);
- }
-
- @Test public void equalsTestwithStringObject() {
- Boolean res = columnIdentifier.equals("String Object");
- assert (!res);
- }
-
- @Test public void equalsTestwithNullColumnId() {
- ColumnIdentifier columnIdentifierTest =
- new ColumnIdentifier(null, columnProperties, DataType.INT);
- Boolean res = columnIdentifierTest.equals(columnIdentifier);
- assert (!res);
- }
-
- @Test public void equalsTestwithDiffColumnId() {
- ColumnIdentifier columnIdentifierTest =
- new ColumnIdentifier("diffColumnId", columnProperties, DataType.INT);
- Boolean res = columnIdentifierTest.equals(columnIdentifier);
- assert (!res);
- }
-
- @Test public void toStringTest() {
- String res = columnIdentifier.toString();
- assert (res.equals("ColumnIdentifier [columnId=columnId]"));
- }
-
- @Test public void getColumnPropertyTest() {
- ColumnIdentifier columnIdentifierTest =
- new ColumnIdentifier("diffColumnId", null, DataType.INT);
- String res = columnIdentifierTest.getColumnProperty("key");
- assertEquals(res, null);
- }
-
- @Test public void getColumnPropertyTestwithNull() {
- assert (columnIdentifier.getColumnProperty("key").equals("value"));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/CompressedDataMeasureWrapperTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/CompressedDataMeasureWrapperTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/CompressedDataMeasureWrapperTest.java
deleted file mode 100644
index 0c3cb84..0000000
--- a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/CompressedDataMeasureWrapperTest.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.carbon.datastorage.filesystem.store.impl;
-
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
-import org.apache.carbondata.core.datastorage.store.impl.CompressedDataMeasureDataWrapper;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static junit.framework.TestCase.assertEquals;
-
-public class CompressedDataMeasureWrapperTest {
- private static CompressedDataMeasureDataWrapper compressedDataMeasureDataWrapper;
- private static CarbonReadDataHolder carbonReadDataHolder;
- private static CarbonReadDataHolder[] carbonReadDataHolders;
-
- @BeforeClass public static void setUp() {
- carbonReadDataHolder = new CarbonReadDataHolder(null);
- carbonReadDataHolders = new CarbonReadDataHolder[] { carbonReadDataHolder };
-
- }
-
- @Test public void testGetValues() {
- compressedDataMeasureDataWrapper = new CompressedDataMeasureDataWrapper(carbonReadDataHolders);
- int expected = 1;
- int actual = compressedDataMeasureDataWrapper.getValues().length;
- assertEquals(expected, actual);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileHolderImplUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileHolderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileHolderImplUnitTest.java
deleted file mode 100644
index 3b3ab9f..0000000
--- a/core/src/test/java/org/apache/carbondata/core/carbon/datastorage/filesystem/store/impl/DFSFileHolderImplUnitTest.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.carbon.datastorage.filesystem.store.impl;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-
-import org.apache.carbondata.core.datastorage.store.impl.DFSFileHolderImpl;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThat;
-import static org.pentaho.di.core.util.Assert.assertNull;
-
-public class DFSFileHolderImplUnitTest {
-
- private static DFSFileHolderImpl dfsFileHolder;
- private static String fileName;
- private static String fileNameWithEmptyContent;
- private static File file;
- private static File fileWithEmptyContent;
-
- @BeforeClass public static void setup() {
- dfsFileHolder = new DFSFileHolderImpl();
- file = new File("Test.carbondata");
- fileWithEmptyContent = new File("TestEXception.carbondata");
-
- if (!file.exists()) try {
- file.createNewFile();
- } catch (IOException e) {
- e.printStackTrace();
- }
- if (!fileWithEmptyContent.exists()) try {
- fileWithEmptyContent.createNewFile();
- } catch (IOException e) {
- e.printStackTrace();
- }
- try {
- FileOutputStream of = new FileOutputStream(file, true);
- BufferedWriter br = new BufferedWriter(new OutputStreamWriter(of, "UTF-8"));
- br.write("Hello World");
- br.close();
- } catch (Exception e) {
- e.getMessage();
- }
- fileName = file.getAbsolutePath();
- fileNameWithEmptyContent = fileWithEmptyContent.getAbsolutePath();
- }
-
- @AfterClass public static void tearDown() throws IOException {
- file.delete();
- fileWithEmptyContent.delete();
- dfsFileHolder.finish();
- }
-
- @Test public void testReadByteArray() throws IOException {
- byte[] result = dfsFileHolder.readByteArray(fileName, 1);
- byte[] expected_result = new byte[] { 72 };
- assertThat(result, is(equalTo(expected_result)));
- }
-
- @Test public void testReadByteArrayWithFilePath() throws IOException {
- byte[] result = dfsFileHolder.readByteArray(fileName, 2L, 2);
- byte[] expected_result = { 108, 108 };
- assertThat(result, is(equalTo(expected_result)));
- }
-
- @Test public void testReadLong() throws IOException {
- long actualResult = dfsFileHolder.readLong(fileName, 1L);
- long expectedResult = 7308335519855243122L;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test(expected = IOException.class)
- public void testReadLongForIoException() throws IOException {
- dfsFileHolder.readLong(fileNameWithEmptyContent, 1L);
- }
-
- @Test(expected = IOException.class)
- public void testReadIntForIoException() throws IOException{
- dfsFileHolder.readInt(fileNameWithEmptyContent, 1L);
- }
-
- @Test public void testReadInt() throws IOException {
- int actualResult = dfsFileHolder.readInt(fileName, 1L);
- int expectedResult = 1701604463;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test public void testReadIntWithFileName() throws IOException {
- int actualResult = dfsFileHolder.readInt(fileName);
- int expectedResult = 1701604463;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test(expected = IOException.class)
- public void testReadIntWithFileNameForIOException() throws IOException {
- dfsFileHolder.readInt(fileNameWithEmptyContent);
- }
-
- @Test public void testDouble() throws IOException {
- double actualResult = dfsFileHolder.readDouble(fileName, 1L);
- double expectedResult = 7.3083355198552433E18;
- assertThat(actualResult, is(equalTo(expectedResult)));
- }
-
- @Test(expected = IOException.class)
- public void testDoubleForIoException() throws IOException {
- dfsFileHolder.readDouble(fileNameWithEmptyContent, 1L);
- }
-
- @Test
- public void testDoubleForIoExceptionwithUpdateCache() throws IOException {
- new MockUp<FileSystem>() {
- @SuppressWarnings("unused") @Mock public FSDataInputStream open(Path file)
- throws IOException {
- throw new IOException();
- }
-
- };
- dfsFileHolder.readDouble(fileName, 1L);
- }
-
-}
[40/57] [abbrv] incubator-carbondata git commit: move load
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 0fecaaf..142796a 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -41,14 +41,14 @@ import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.block.{Distributable, TableBlockInfo}
import org.apache.carbondata.core.dictionary.server.DictionaryServer
-import org.apache.carbondata.core.load.{BlockDetails, LoadMetadataDetails}
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.metadata.{CarbonTableIdentifier, ColumnarFormatVersion}
import org.apache.carbondata.core.path.CarbonStorePath
import org.apache.carbondata.core.update.CarbonUpdateUtil
+import org.apache.carbondata.core.updatestatus.LoadMetadataDetails
import org.apache.carbondata.core.updatestatus.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.processing.csvreaderstep.RddInpututilsForUpdate
+import org.apache.carbondata.processing.csvreaderstep.{BlockDetails, RddInpututilsForUpdate}
import org.apache.carbondata.processing.etl.DataLoadingException
import org.apache.carbondata.processing.model.CarbonLoadModel
import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index 441500b..db67940 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -40,14 +40,14 @@ import org.apache.spark.{SparkEnv, SparkException}
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.block.{Distributable, TableBlockInfo}
-import org.apache.carbondata.core.load.{BlockDetails, LoadMetadataDetails}
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.metadata.{CarbonTableIdentifier, ColumnarFormatVersion}
import org.apache.carbondata.core.path.CarbonStorePath
import org.apache.carbondata.core.update.CarbonUpdateUtil
+import org.apache.carbondata.core.updatestatus.LoadMetadataDetails
import org.apache.carbondata.core.updatestatus.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.processing.csvreaderstep.RddInpututilsForUpdate
+import org.apache.carbondata.processing.csvreaderstep.{BlockDetails, RddInpututilsForUpdate}
import org.apache.carbondata.processing.etl.DataLoadingException
import org.apache.carbondata.processing.model.CarbonLoadModel
import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/BlockDetails.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/BlockDetails.java b/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/BlockDetails.java
new file mode 100644
index 0000000..cc6ae5f
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/BlockDetails.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.processing.csvreaderstep;
+
+import java.io.Serializable;
+
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+
+/**
+ * blocks info
+ * TODO Remove this class after removing of kettle.
+ */
+public class BlockDetails extends FileSplit implements Serializable {
+
+ /**
+ * serialization version
+ */
+ private static final long serialVersionUID = 2293906691860002339L;
+ //block offset
+ private long blockOffset;
+ //block length
+ private long blockLength;
+ //file path which block belong to
+ private String filePath;
+ // locations where this block exists
+ private String[] locations;
+
+ public BlockDetails(Path filePath, long blockOffset, long blockLength, String[] locations) {
+ super(filePath, blockOffset, blockLength, locations);
+ this.filePath = filePath.toString();
+ this.blockOffset = blockOffset;
+ this.blockLength = blockLength;
+ this.locations = locations;
+ }
+
+ public long getBlockOffset() {
+ return blockOffset;
+ }
+
+ public long getBlockLength() {
+ return blockLength;
+ }
+
+ public String getFilePath() {
+ return FileFactory.getUpdatedFilePath(filePath);
+ }
+
+ public void setFilePath(String filePath) {
+ this.filePath = filePath;
+ }
+
+ public String[] getLocations() {
+ return locations;
+ }
+
+ /** The file containing this split's data. */
+ @Override
+ public Path getPath() { return new Path(filePath); }
+
+ /** The position of the first byte in the file to process. */
+ @Override
+ public long getStart() { return blockOffset; }
+
+ /** The number of bytes in the file to process. */
+ @Override
+ public long getLength() { return blockLength; }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/CsvInput.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/CsvInput.java b/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/CsvInput.java
index 48efec8..742783f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/CsvInput.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/CsvInput.java
@@ -34,7 +34,6 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.common.logging.impl.StandardLogService;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.load.BlockDetails;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
import org.apache.carbondata.processing.graphgenerator.GraphGenerator;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/UnivocityCsvParserVo.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/UnivocityCsvParserVo.java b/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/UnivocityCsvParserVo.java
index 5383309..2663d0c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/UnivocityCsvParserVo.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/UnivocityCsvParserVo.java
@@ -21,8 +21,6 @@ package org.apache.carbondata.processing.csvreaderstep;
import java.util.List;
-import org.apache.carbondata.core.load.BlockDetails;
-
/**
* Vo class which will holds all the properties required to read and parse the
* csv file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java b/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java
index 676f702..d26bf12 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java
@@ -44,7 +44,7 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.load.BlockDetails;
+import org.apache.carbondata.processing.csvreaderstep.BlockDetails;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.processing.api.dataloader.DataLoadModel;
import org.apache.carbondata.processing.api.dataloader.SchemaInfo;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
index 8be315e..7a21b9e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
@@ -27,7 +27,7 @@ import java.util.HashMap;
import java.util.List;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.load.LoadMetadataDetails;
+import org.apache.carbondata.core.updatestatus.LoadMetadataDetails;
import org.apache.carbondata.core.update.SegmentUpdateDetails;
import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java b/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
index a105eee..59fdda8 100644
--- a/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
+++ b/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
@@ -46,8 +46,8 @@ import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.impl.FileFactory;
-import org.apache.carbondata.core.load.BlockDetails;
-import org.apache.carbondata.core.load.LoadMetadataDetails;
+import org.apache.carbondata.processing.csvreaderstep.BlockDetails;
+import org.apache.carbondata.core.updatestatus.LoadMetadataDetails;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.writer.CarbonDictionaryWriter;
[42/57] [abbrv] incubator-carbondata git commit: move load
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
index d2503d3..b1a01f8 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
@@ -30,14 +30,14 @@ import java.util.concurrent.TimeUnit;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.metadata.CarbonMetadata;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.devapi.BiDictionary;
import org.apache.carbondata.core.devapi.DictionaryGenerationException;
import org.apache.carbondata.core.devapi.DictionaryGenerator;
import org.apache.carbondata.core.dictionary.generator.key.DictionaryKey;
+import org.apache.carbondata.core.metadata.CarbonMetadata;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.util.CarbonProperties;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServerHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServerHandler.java b/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServerHandler.java
index 9160ffe..2ad15fd 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServerHandler.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServerHandler.java
@@ -24,7 +24,11 @@ import org.apache.carbondata.core.dictionary.generator.ServerDictionaryGenerator
import org.apache.carbondata.core.dictionary.generator.key.DictionaryKey;
import org.apache.carbondata.core.dictionary.generator.key.KryoRegister;
-import org.jboss.netty.channel.*;
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.channel.ChannelStateEvent;
+import org.jboss.netty.channel.ExceptionEvent;
+import org.jboss.netty.channel.MessageEvent;
+import org.jboss.netty.channel.SimpleChannelHandler;
/**
* Handler for Dictionary server.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactory.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactory.java
index c9edf9a..7db989a 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactory.java
@@ -18,10 +18,12 @@
*/
package org.apache.carbondata.core.keygenerator.directdictionary;
-import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.keygenerator.directdictionary.timestamp.DateDirectDictionaryGenerator;
-import org.apache.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampDirectDictionaryGenerator;
+import org.apache.carbondata.core.keygenerator.directdictionary.timestamp
+ .DateDirectDictionaryGenerator;
+import org.apache.carbondata.core.keygenerator.directdictionary.timestamp
+ .TimeStampDirectDictionaryGenerator;
+import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.util.CarbonProperties;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
index 8b97bea..44569de 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/DateDirectDictionaryGenerator.java
@@ -24,9 +24,9 @@ import java.util.Date;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
+import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.util.CarbonProperties;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
index aa37fa3..cd6449a 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/timestamp/TimeStampDirectDictionaryGenerator.java
@@ -24,15 +24,19 @@ import java.util.Date;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
+import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.util.CarbonProperties;
-import static org.apache.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampGranularityConstants.TIME_GRAN_DAY;
-import static org.apache.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampGranularityConstants.TIME_GRAN_HOUR;
-import static org.apache.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampGranularityConstants.TIME_GRAN_MIN;
-import static org.apache.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampGranularityConstants.TIME_GRAN_SEC;
+import static org.apache.carbondata.core.keygenerator.directdictionary.timestamp
+ .TimeStampGranularityConstants.TIME_GRAN_DAY;
+import static org.apache.carbondata.core.keygenerator.directdictionary.timestamp
+ .TimeStampGranularityConstants.TIME_GRAN_HOUR;
+import static org.apache.carbondata.core.keygenerator.directdictionary.timestamp
+ .TimeStampGranularityConstants.TIME_GRAN_MIN;
+import static org.apache.carbondata.core.keygenerator.directdictionary.timestamp
+ .TimeStampGranularityConstants.TIME_GRAN_SEC;
/**
* The class provides the method to generate dictionary key and getting the actual value from
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/load/BlockDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/load/BlockDetails.java b/core/src/main/java/org/apache/carbondata/core/load/BlockDetails.java
deleted file mode 100644
index 5f00f23..0000000
--- a/core/src/main/java/org/apache/carbondata/core/load/BlockDetails.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.load;
-
-import java.io.Serializable;
-
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.lib.input.FileSplit;
-
-/**
- * blocks info
- * TODO Remove this class after removing of kettle.
- */
-public class BlockDetails extends FileSplit implements Serializable {
-
- /**
- * serialization version
- */
- private static final long serialVersionUID = 2293906691860002339L;
- //block offset
- private long blockOffset;
- //block length
- private long blockLength;
- //file path which block belong to
- private String filePath;
- // locations where this block exists
- private String[] locations;
-
- public BlockDetails(Path filePath, long blockOffset, long blockLength, String[] locations) {
- super(filePath, blockOffset, blockLength, locations);
- this.filePath = filePath.toString();
- this.blockOffset = blockOffset;
- this.blockLength = blockLength;
- this.locations = locations;
- }
-
- public long getBlockOffset() {
- return blockOffset;
- }
-
- public long getBlockLength() {
- return blockLength;
- }
-
- public String getFilePath() {
- return FileFactory.getUpdatedFilePath(filePath);
- }
-
- public void setFilePath(String filePath) {
- this.filePath = filePath;
- }
-
- public String[] getLocations() {
- return locations;
- }
-
- /** The file containing this split's data. */
- @Override
- public Path getPath() { return new Path(filePath); }
-
- /** The position of the first byte in the file to process. */
- @Override
- public long getStart() { return blockOffset; }
-
- /** The number of bytes in the file to process. */
- @Override
- public long getLength() { return blockLength; }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/load/LoadMetadataDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/load/LoadMetadataDetails.java b/core/src/main/java/org/apache/carbondata/core/load/LoadMetadataDetails.java
deleted file mode 100644
index 2861b20..0000000
--- a/core/src/main/java/org/apache/carbondata/core/load/LoadMetadataDetails.java
+++ /dev/null
@@ -1,344 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.load;
-
-import java.io.Serializable;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-
-public class LoadMetadataDetails implements Serializable {
-
- private static final long serialVersionUID = 1106104914918491724L;
- private String timestamp;
- private String loadStatus;
- private String loadName;
- private String partitionCount;
- private String isDeleted = CarbonCommonConstants.KEYWORD_FALSE;
-
- // update delta end timestamp
- private String updateDeltaEndTimestamp = "";
-
- // update delta start timestamp
- private String updateDeltaStartTimestamp = "";
-
- // this will represent the update status file name at that point of time.
- private String updateStatusFileName = "";
-
- /**
- * LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(LoadMetadataDetails.class.getName());
-
- // dont remove static as the write will fail.
- private static final SimpleDateFormat parser =
- new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP);
- /**
- * Segment modification or deletion time stamp
- */
- private String modificationOrdeletionTimesStamp;
- private String loadStartTime;
-
- private String mergedLoadName;
- /**
- * visibility is used to determine whether to the load is visible or not.
- */
- private String visibility = "true";
-
- /**
- * To know if the segment is a major compacted segment or not.
- */
- private String majorCompacted;
-
- public String getPartitionCount() {
- return partitionCount;
- }
-
- public void setPartitionCount(String partitionCount) {
- this.partitionCount = partitionCount;
- }
-
- public long getLoadEndTime() {
- return convertTimeStampToLong(timestamp);
- }
-
- public void setLoadEndTime(long timestamp) {
- this.timestamp = getTimeStampConvertion(timestamp);;
- }
-
- public String getLoadStatus() {
- return loadStatus;
- }
-
- public void setLoadStatus(String loadStatus) {
- this.loadStatus = loadStatus;
- }
-
- public String getLoadName() {
- return loadName;
- }
-
- public void setLoadName(String loadName) {
- this.loadName = loadName;
- }
-
- /**
- * @return the modificationOrdeletionTimesStamp
- */
- public long getModificationOrdeletionTimesStamp() {
- if(null == modificationOrdeletionTimesStamp) {
- return 0;
- }
- return convertTimeStampToLong(modificationOrdeletionTimesStamp);
- }
-
- /**
- * @param modificationOrdeletionTimesStamp the modificationOrdeletionTimesStamp to set
- */
- public void setModificationOrdeletionTimesStamp(long modificationOrdeletionTimesStamp) {
- this.modificationOrdeletionTimesStamp =
- getTimeStampConvertion(modificationOrdeletionTimesStamp);
- }
-
- /* (non-Javadoc)
- * @see java.lang.Object#hashCode()
- */
- @Override public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((loadName == null) ? 0 : loadName.hashCode());
- return result;
- }
-
- /* (non-Javadoc)
- * @see java.lang.Object#equals(java.lang.Object)
- */
- @Override public boolean equals(Object obj) {
- if (obj == null) {
- return false;
-
- }
- if (!(obj instanceof LoadMetadataDetails)) {
- return false;
- }
- LoadMetadataDetails other = (LoadMetadataDetails) obj;
- if (loadName == null) {
- if (other.loadName != null) {
- return false;
- }
- } else if (!loadName.equals(other.loadName)) {
- return false;
- }
- return true;
- }
-
- /**
- * @return the startLoadTime
- */
- public long getLoadStartTime() {
- return convertTimeStampToLong(loadStartTime);
- }
-
- /**
- * return loadStartTime
- *
- * @return
- */
- public long getLoadStartTimeAsLong() {
- return (!loadStartTime.isEmpty()) ? getTimeStamp(loadStartTime) : 0;
- }
-
- /**
- * This method will convert a given timestamp to long value and then to string back
- *
- * @param factTimeStamp
- * @return
- */
- private long convertTimeStampToLong(String factTimeStamp) {
- SimpleDateFormat parser = new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP_MILLIS);
- Date dateToStr = null;
- try {
- dateToStr = parser.parse(factTimeStamp);
- return dateToStr.getTime();
- } catch (ParseException e) {
- LOGGER.error("Cannot convert" + factTimeStamp + " to Time/Long type value" + e.getMessage());
- parser = new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP);
- try {
- dateToStr = parser.parse(factTimeStamp);
- return dateToStr.getTime();
- } catch (ParseException e1) {
- LOGGER
- .error("Cannot convert" + factTimeStamp + " to Time/Long type value" + e1.getMessage());
- return 0;
- }
- }
- }
-
- /**
- * returns load start time as long value
- *
- * @param loadStartTime
- * @return
- */
- public Long getTimeStamp(String loadStartTime) {
- Date dateToStr = null;
- try {
- dateToStr = parser.parse(loadStartTime);
- return dateToStr.getTime() * 1000;
- } catch (ParseException e) {
- LOGGER.error("Cannot convert" + loadStartTime + " to Time/Long type value" + e.getMessage());
- return null;
- }
- }
-
- private String getTimeStampConvertion(long time) {
- SimpleDateFormat sdf = new SimpleDateFormat(CarbonCommonConstants.CARBON_TIMESTAMP_MILLIS);
- return sdf.format(time);
- }
-
- /**
- * @param loadStartTime
- */
- public void setLoadStartTime(long loadStartTime) {
- this.loadStartTime = getTimeStampConvertion(loadStartTime);
- }
-
- /**
- * @return the mergedLoadName
- */
- public String getMergedLoadName() {
- return mergedLoadName;
- }
-
- /**
- * @param mergedLoadName the mergedLoadName to set
- */
- public void setMergedLoadName(String mergedLoadName) {
- this.mergedLoadName = mergedLoadName;
- }
-
- /**
- * @return the visibility
- */
- public String getVisibility() {
- return visibility;
- }
-
- /**
- * @param visibility the visibility to set
- */
- public void setVisibility(String visibility) {
- this.visibility = visibility;
- }
-
- /**
- * Return true if it is a major compacted segment.
- * @return majorCompacted
- */
- public String isMajorCompacted() {
- return majorCompacted;
- }
-
- /**
- * Set true if it is a major compacted segment.
- *
- * @param majorCompacted
- */
- public void setMajorCompacted(String majorCompacted) {
- this.majorCompacted = majorCompacted;
- }
-
- /**
- * To get isDeleted property.
- *
- * @return isDeleted
- */
- public String getIsDeleted() {
- return isDeleted;
- }
-
- /**
- * To set isDeleted property.
- *
- * @param isDeleted
- */
- public void setIsDeleted(String isDeleted) {
- this.isDeleted = isDeleted;
- }
-
- /**
- * To get the update delta end timestamp
- *
- * @return updateDeltaEndTimestamp
- */
- public String getUpdateDeltaEndTimestamp() {
- return updateDeltaEndTimestamp;
- }
-
- /**
- * To set the update delta end timestamp
- *
- * @param updateDeltaEndTimestamp
- */
- public void setUpdateDeltaEndTimestamp(String updateDeltaEndTimestamp) {
- this.updateDeltaEndTimestamp = updateDeltaEndTimestamp;
- }
-
- /**
- * To get the update delta start timestamp
- *
- * @return updateDeltaStartTimestamp
- */
- public String getUpdateDeltaStartTimestamp() {
- return updateDeltaStartTimestamp;
- }
-
- /**
- * To set the update delta start timestamp
- *
- * @param updateDeltaStartTimestamp
- */
- public void setUpdateDeltaStartTimestamp(String updateDeltaStartTimestamp) {
- this.updateDeltaStartTimestamp = updateDeltaStartTimestamp;
- }
-
- /**
- * To get the updateStatusFileName
- *
- * @return updateStatusFileName
- */
- public String getUpdateStatusFileName() {
- return updateStatusFileName;
- }
-
- /**
- * To set the updateStatusFileName
- *
- * @param updateStatusFileName
- */
- public void setUpdateStatusFileName(String updateStatusFileName) {
- this.updateStatusFileName = updateStatusFileName;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java b/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java
index c5a6087..62d5bdb 100644
--- a/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java
+++ b/core/src/main/java/org/apache/carbondata/core/memory/HeapMemoryAllocator.java
@@ -17,11 +17,11 @@
package org.apache.carbondata.core.memory;
+import javax.annotation.concurrent.GuardedBy;
import java.lang.ref.WeakReference;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
-import javax.annotation.concurrent.GuardedBy;
import org.apache.carbondata.core.unsafe.CarbonUnsafe;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/DataFileFooter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/DataFileFooter.java b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/DataFileFooter.java
index d064ab3..7b9d1e2 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/DataFileFooter.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/DataFileFooter.java
@@ -21,8 +21,8 @@ package org.apache.carbondata.core.metadata.blocklet;
import java.io.Serializable;
import java.util.List;
-import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.datastore.block.BlockInfo;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index dae3c26..d5cbe6a 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -20,10 +20,17 @@
package org.apache.carbondata.core.metadata.schema.table;
import java.io.Serializable;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.Encoding;
@@ -33,7 +40,6 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonImplicitDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
/**
* Mapping class for Carbon actual table
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
index 3dff56c..b350469 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
@@ -23,10 +23,10 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.schema.BucketingInfo;
import org.apache.carbondata.core.metadata.schema.SchemaEvolution;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
/**
* Persisting the table information
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/path/CarbonTablePath.java
index 45c947f..7fb77c1 100644
--- a/core/src/main/java/org/apache/carbondata/core/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/path/CarbonTablePath.java
@@ -20,15 +20,16 @@ package org.apache.carbondata.core.path;
import java.io.File;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastorage.impl.FileFactory;
-import static org.apache.carbondata.core.constants.CarbonCommonConstants.INVALID_SEGMENT_ID;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.hadoop.fs.Path;
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.INVALID_SEGMENT_ID;
+
/**
* Helps to get Table content paths.
*/
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java
index 097c25b..c313080 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java
@@ -19,16 +19,20 @@
package org.apache.carbondata.core.reader;
-import java.io.*;
+import java.io.BufferedReader;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.StringWriter;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.atomic.AtomicFileOperations;
+import org.apache.carbondata.core.atomic.AtomicFileOperationsImpl;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.update.DeleteDeltaBlockDetails;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.atomic.AtomicFileOperations;
-import org.apache.carbondata.core.atomic.AtomicFileOperationsImpl;
import com.google.gson.Gson;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteFilesDataReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteFilesDataReader.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteFilesDataReader.java
index 04f6dfd..4d3c9cc 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteFilesDataReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteFilesDataReader.java
@@ -24,7 +24,11 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
-import java.util.concurrent.*;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
index 318836c..1e30236 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryMetadataReaderImpl.java
@@ -23,11 +23,11 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.carbondata.core.service.CarbonCommonFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.path.CarbonTablePath;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.core.service.PathService;
import org.apache.carbondata.format.ColumnDictionaryChunkMeta;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java
index 76b22cb..0f931fd 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImpl.java
@@ -25,11 +25,11 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.core.cache.dictionary.ColumnDictionaryChunkIterator;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.path.CarbonTablePath;
+import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.core.service.PathService;
import org.apache.carbondata.format.ColumnDictionaryChunk;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
index 1b95507..4472f4d 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
@@ -21,17 +21,17 @@ package org.apache.carbondata.core.reader.sortindex;
import java.io.IOException;
import java.util.List;
-import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.path.CarbonTablePath;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk;
import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader;
import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReaderImpl;
import org.apache.carbondata.core.reader.ThriftReader;
+import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.core.service.PathService;
import org.apache.carbondata.format.ColumnSortInfo;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
index 3f7dd4a..7f0c073 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/AbstractScannedResultCollector.java
@@ -23,8 +23,8 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
-import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.scan.collector.ScannedResultCollector;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
index c62def7..e0efdc0 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedResultCollector.java
@@ -24,19 +24,19 @@ import java.util.Arrays;
import java.util.List;
import java.util.Map;
-import org.apache.carbondata.core.update.data.BlockletLevelDeleteDeltaDataCache;
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.filter.GenericQueryType;
import org.apache.carbondata.core.scan.model.QueryDimension;
import org.apache.carbondata.core.scan.model.QueryMeasure;
import org.apache.carbondata.core.scan.result.AbstractScannedResult;
+import org.apache.carbondata.core.update.data.BlockletLevelDeleteDeltaDataCache;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.commons.lang3.ArrayUtils;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java
index e4d6d68..a0ba393 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/DictionaryBasedVectorResultCollector.java
@@ -22,8 +22,8 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
+import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.model.QueryDimension;
import org.apache.carbondata.core.scan.model.QueryMeasure;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java
index cf42d9c..0ea3e9f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/collector/impl/RawBasedResultCollector.java
@@ -22,12 +22,12 @@ import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
-import org.apache.carbondata.core.update.data.BlockletLevelDeleteDeltaDataCache;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.model.QueryMeasure;
import org.apache.carbondata.core.scan.result.AbstractScannedResult;
import org.apache.carbondata.core.scan.wrappers.ByteArrayWrapper;
+import org.apache.carbondata.core.update.data.BlockletLevelDeleteDeltaDataCache;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java
index 9ef5fad..3dff9e6 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/complextypes/PrimitiveQueryType.java
@@ -29,9 +29,9 @@ import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionary
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
import org.apache.carbondata.core.keygenerator.mdkey.Bits;
import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.core.scan.filter.GenericQueryType;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.util.DataTypeUtil;
public class PrimitiveQueryType extends ComplexQueryType implements GenericQueryType {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
index d2da9d3..9eedb75 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/AbstractQueryExecutor.java
@@ -33,25 +33,21 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.common.logging.impl.StandardLogService;
import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.cache.CacheType;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.BlockIndexStore;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.block.TableBlockUniqueIdentifier;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.keygenerator.KeyGenerator;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.path.CarbonStorePath;
-import org.apache.carbondata.core.stats.QueryStatistic;
-import org.apache.carbondata.core.stats.QueryStatisticsConstants;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
-import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
-import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.scan.executor.QueryExecutor;
import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
import org.apache.carbondata.core.scan.executor.infos.AggregatorInfo;
@@ -63,6 +59,10 @@ import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.model.QueryDimension;
import org.apache.carbondata.core.scan.model.QueryMeasure;
import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.core.stats.QueryStatistic;
+import org.apache.carbondata.core.stats.QueryStatisticsConstants;
+import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
+import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.commons.lang3.ArrayUtils;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/QueryExecutorProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/QueryExecutorProperties.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/QueryExecutorProperties.java
index b4c72e7..30ca98a 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/QueryExecutorProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/impl/QueryExecutorProperties.java
@@ -28,8 +28,8 @@ import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-import org.apache.carbondata.core.stats.QueryStatisticsRecorder;
import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
+import org.apache.carbondata.core.stats.QueryStatisticsRecorder;
/**
* Holds all the properties required for query execution
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
index 6f4a37e..c8f4e7c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
@@ -21,11 +21,11 @@ package org.apache.carbondata.core.scan.executor.infos;
import java.util.Map;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.scan.filter.GenericQueryType;
import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
import org.apache.carbondata.core.scan.model.QueryDimension;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
index a9c6df0..db09374 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
@@ -37,19 +37,18 @@ import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.cache.CacheType;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.keygenerator.KeyGenerator;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonMetadata;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
-import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.scan.complextypes.ArrayQueryType;
import org.apache.carbondata.core.scan.complextypes.PrimitiveQueryType;
import org.apache.carbondata.core.scan.complextypes.StructQueryType;
@@ -62,6 +61,7 @@ import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnRes
import org.apache.carbondata.core.scan.model.QueryDimension;
import org.apache.carbondata.core.scan.model.QueryMeasure;
import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.commons.lang3.ArrayUtils;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
index 67835b6..8cb5aeb 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
@@ -21,10 +21,10 @@ package org.apache.carbondata.core.scan.executor.util;
import java.util.ArrayList;
import java.util.List;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.scan.executor.infos.AggregatorInfo;
import org.apache.carbondata.core.scan.model.QueryDimension;
import org.apache.carbondata.core.scan.model.QueryMeasure;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java b/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java
index c3a0949..a95200f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/expression/ExpressionResult.java
@@ -27,10 +27,10 @@ import java.util.ArrayList;
import java.util.Date;
import java.util.List;
-import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.util.CarbonProperties;
public class ExpressionResult implements Comparable<ExpressionResult> {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
index 31321ec..94b8273 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
@@ -25,15 +25,15 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.datastore.DataRefNodeFinder;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.impl.btree.BTreeDataRefNodeFinder;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.metadata.Encoding;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.scan.expression.BinaryExpression;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.conditional.BinaryConditionalExpression;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java
index 038f496..7eadc62 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterProcessor.java
@@ -22,9 +22,9 @@ package org.apache.carbondata.core.scan.filter;
import java.io.IOException;
import java.util.List;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index 9af8d53..131d19f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -46,19 +46,15 @@ import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryChunksWrapper;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
import org.apache.carbondata.core.cache.dictionary.ForwardDictionary;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.keygenerator.KeyGenException;
+import org.apache.carbondata.core.keygenerator.KeyGenerator;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
-import org.apache.carbondata.core.util.ByteUtil;
-import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.ExpressionResult;
@@ -84,6 +80,10 @@ import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.carbondata.core.scan.filter.resolver.RowLevelFilterResolverImpl;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
public final class FilterUtil {
private static final LogService LOGGER =
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
index 47e9f09..ddc4038 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeColGroupFilterExecuterImpl.java
@@ -28,10 +28,10 @@ import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
-import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.executor.util.QueryUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.util.ByteUtil;
/**
* It checks if filter is required on given block and if required, it does
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
index e8c9903..c78998c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/ExcludeFilterExecuterImpl.java
@@ -25,10 +25,10 @@ import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
-import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.util.CarbonUtil;
public class ExcludeFilterExecuterImpl implements FilterExecuter {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
index 22aa3c9..5c5c1a0 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeColGroupFilterExecuterImpl.java
@@ -28,10 +28,10 @@ import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
-import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.executor.util.QueryUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.util.ByteUtil;
/**
* It checks if filter is required on given block and if required, it does
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
index 4268966..5fc23b7 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/IncludeFilterExecuterImpl.java
@@ -25,11 +25,11 @@ import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
-import org.apache.carbondata.core.util.ByteUtil;
-import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.CarbonUtil;
public class IncludeFilterExecuterImpl implements FilterExecuter {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index a47704d..2fd2bb0 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -31,17 +31,15 @@ import java.util.Map;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.core.metadata.Encoding;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.executor.util.QueryUtil;
import org.apache.carbondata.core.scan.expression.Expression;
@@ -54,6 +52,8 @@ import org.apache.carbondata.core.scan.filter.intf.RowIntf;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
public class RowLevelFilterExecuterImpl implements FilterExecuter {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
index dfab0b1..7e09155 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
@@ -22,18 +22,18 @@ import java.io.IOException;
import java.util.BitSet;
import java.util.List;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.Encoding;
-import org.apache.carbondata.core.util.ByteUtil;
-import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.CarbonUtil;
public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecuterImpl {
private byte[][] filterRangeValues;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
index 825f59e..94758dc 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtrThanEquaToFilterExecuterImpl.java
@@ -22,18 +22,18 @@ import java.io.IOException;
import java.util.BitSet;
import java.util.List;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.Encoding;
-import org.apache.carbondata.core.util.ByteUtil;
-import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.CarbonUtil;
public class RowLevelRangeGrtrThanEquaToFilterExecuterImpl extends RowLevelFilterExecuterImpl {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
index 1eb47e6..20d7510 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanEqualFilterExecuterImpl.java
@@ -22,21 +22,21 @@ import java.io.IOException;
import java.util.BitSet;
import java.util.List;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
-import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
-import org.apache.carbondata.core.util.ByteUtil;
-import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.CarbonUtil;
public class RowLevelRangeLessThanEqualFilterExecuterImpl extends RowLevelFilterExecuterImpl {
protected byte[][] filterRangeValues;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
index 83fa050..cd25fb5 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeLessThanFiterExecuterImpl.java
@@ -22,21 +22,21 @@ import java.io.IOException;
import java.util.BitSet;
import java.util.List;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
-import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
-import org.apache.carbondata.core.util.ByteUtil;
-import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.util.CarbonUtil;
public class RowLevelRangeLessThanFiterExecuterImpl extends RowLevelFilterExecuterImpl {
private byte[][] filterRangeValues;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
index 69cf216..967ee5c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
@@ -22,8 +22,8 @@ import java.io.IOException;
import java.util.List;
import java.util.SortedMap;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
@@ -36,7 +36,8 @@ import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType;
import org.apache.carbondata.core.scan.filter.resolver.metadata.FilterResolverMetadata;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.visitor.FilterInfoTypeVisitorFactory;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.visitor
+ .FilterInfoTypeVisitorFactory;
public class ConditionalFilterResolverImpl implements FilterResolverIntf {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java
index 9b42ecc..1c8eb80 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/FilterResolverIntf.java
@@ -23,8 +23,8 @@ import java.io.Serializable;
import java.util.List;
import java.util.SortedMap;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java
index 11f9eaa..6f2bb4f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/LogicalFilterResolverImpl.java
@@ -22,8 +22,8 @@ package org.apache.carbondata.core.scan.filter.resolver;
import java.util.List;
import java.util.SortedMap;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.scan.expression.BinaryExpression;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
index ebf598f..b2fe42c 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelFilterResolverImpl.java
@@ -22,8 +22,8 @@ package org.apache.carbondata.core.scan.filter.resolver;
import java.util.ArrayList;
import java.util.List;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.conditional.ConditionalExpression;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
index 88dfab1..c85b88f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
@@ -25,13 +25,12 @@ import java.util.Comparator;
import java.util.List;
import java.util.SortedMap;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
-import org.apache.carbondata.core.util.ByteUtil;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.ExpressionResult;
@@ -44,6 +43,7 @@ import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.MeasureColumnResolvedFilterInfo;
+import org.apache.carbondata.core.util.ByteUtil;
public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverImpl {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
index 450674c..1224ef5 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/resolverinfo/visitor/CustomTypeDictionaryVisitor.java
@@ -25,13 +25,13 @@ import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
-import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
import org.apache.carbondata.core.scan.filter.DimColumnFilterInfo;
import org.apache.carbondata.core.scan.filter.resolver.metadata.FilterResolverMetadata;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.util.CarbonProperties;
public class CustomTypeDictionaryVisitor implements ResolvedFilterInfoVisitorIntf {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
index 41afa4b..e88b185 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/model/QueryModel.java
@@ -26,21 +26,21 @@ import java.util.List;
import java.util.Map;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-import org.apache.carbondata.core.stats.QueryStatisticsRecorder;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.update.data.UpdateVO;
-import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.scan.expression.ColumnExpression;
import org.apache.carbondata.core.scan.expression.Expression;
import org.apache.carbondata.core.scan.expression.UnknownExpression;
import org.apache.carbondata.core.scan.expression.conditional.ConditionalExpression;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+import org.apache.carbondata.core.stats.QueryStatisticsRecorder;
+import org.apache.carbondata.core.update.data.UpdateVO;
+import org.apache.carbondata.core.util.CarbonUtil;
/**
* Query model which will have all the detail
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
index dbbb6dc..fcfd924 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
@@ -24,9 +24,8 @@ import java.util.List;
import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.DataRefNode;
-import org.apache.carbondata.core.stats.QueryStatisticsModel;
import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.scan.collector.ScannedResultCollector;
import org.apache.carbondata.core.scan.collector.impl.DictionaryBasedResultCollector;
import org.apache.carbondata.core.scan.collector.impl.DictionaryBasedVectorResultCollector;
@@ -38,6 +37,7 @@ import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
import org.apache.carbondata.core.scan.scanner.BlockletScanner;
import org.apache.carbondata.core.scan.scanner.impl.FilterScanner;
import org.apache.carbondata.core.scan.scanner.impl.NonFilterScanner;
+import org.apache.carbondata.core.stats.QueryStatisticsModel;
/**
* This abstract class provides a skeletal implementation of the
[36/57] [abbrv] incubator-carbondata git commit: move identifier to
metadata package
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
index 0394d31..11eb0c8 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonIUDMergerRDD.scala
@@ -27,8 +27,9 @@ import org.apache.hadoop.mapreduce.Job
import org.apache.spark.{Partition, SparkContext}
import org.apache.spark.sql.execution.command.CarbonMergerMapping
-import org.apache.carbondata.core.{AbsoluteTableIdentifier, CarbonTableIdentifier}
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier
import org.apache.carbondata.core.datastore.block.{Distributable, TableBlockInfo}
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.hadoop.{CarbonInputFormat, CarbonInputSplit, CarbonMultiBlockSplit}
import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil
import org.apache.carbondata.processing.model.CarbonLoadModel
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index 4acef95..523ed96 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -34,22 +34,22 @@ import org.apache.spark.sql.execution.command.{CarbonMergerMapping, NodeInfo}
import org.apache.spark.sql.hive.DistributionUtil
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.{AbsoluteTableIdentifier, CarbonTableIdentifier}
-import org.apache.carbondata.core.datastore.block.{Distributable, SegmentProperties, TableBlockInfo, TableTaskInfo, TaskBlockInfo}
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.block._
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.core.path.CarbonTablePath
-import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.scan.result.iterator.RawResultIterator
+import org.apache.carbondata.core.update.data.UpdateVO
import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
-import org.apache.carbondata.hadoop.{CarbonInputFormat, CarbonInputSplit, CarbonMultiBlockSplit}
import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil
+import org.apache.carbondata.hadoop.{CarbonInputFormat, CarbonInputSplit, CarbonMultiBlockSplit}
import org.apache.carbondata.processing.model.CarbonLoadModel
import org.apache.carbondata.processing.util.CarbonDataProcessorUtil
-import org.apache.carbondata.core.scan.result.iterator.RawResultIterator
-import org.apache.carbondata.core.update.data.UpdateVO
import org.apache.carbondata.spark.MergeResult
import org.apache.carbondata.spark.load.CarbonLoaderUtil
-import org.apache.carbondata.spark.merger.{CarbonCompactionExecutor, CarbonCompactionUtil, CarbonDataMergerUtil, CompactionType, RowResultMerger}
+import org.apache.carbondata.spark.merger.{CarbonCompactionExecutor, CarbonCompactionUtil, CompactionType, RowResultMerger}
import org.apache.carbondata.spark.splits.TableSplit
class CarbonMergerRDD[K, V](
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
index 2ac8ee3..c7bbfa9 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonScanRDD.scala
@@ -31,8 +31,8 @@ import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.hive.DistributionUtil
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.AbsoluteTableIdentifier
import org.apache.carbondata.core.datastore.block.Distributable
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.stats.{QueryStatistic, QueryStatisticsConstants, QueryStatisticsRecorder}
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
index 8e80ab5..de57639 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
@@ -27,15 +27,14 @@ import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.execution.command.{CompactionCallableModel, CompactionModel}
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.{CarbonDataLoadSchema, CarbonTableIdentifier}
-import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.metadata.{CarbonMetadata, CarbonTableIdentifier}
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.load.LoadMetadataDetails
import org.apache.carbondata.core.update.CarbonUpdateUtil
import org.apache.carbondata.core.updatestatus.SegmentStatusManager
-import org.apache.carbondata.locks.{CarbonLockFactory, CarbonLockUtil, LockUsage}
-import org.apache.carbondata.processing.model.CarbonLoadModel
+import org.apache.carbondata.core.updatestatus.locks.{CarbonLockFactory, CarbonLockUtil, LockUsage}
+import org.apache.carbondata.processing.model.{CarbonDataLoadSchema, CarbonLoadModel}
import org.apache.carbondata.spark._
import org.apache.carbondata.spark.load._
import org.apache.carbondata.spark.merger.{CarbonDataMergerUtil, CompactionCallable, CompactionType}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
index 013cb9b..708e9b6 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -37,10 +37,8 @@ import org.apache.spark.sql.types.{StringType, StructField, StructType}
import org.apache.spark.util.FileUtils
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.CarbonTableIdentifier
import org.apache.carbondata.core.cache.dictionary.Dictionary
-import org.apache.carbondata.core.metadata.DataType
-import org.apache.carbondata.core.metadata.Encoding
+import org.apache.carbondata.core.metadata.{CarbonTableIdentifier, DataType, Encoding}
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.path.CarbonStorePath
import org.apache.carbondata.core.constants.CarbonCommonConstants
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java b/integration/spark/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
index b168ecc..180acc2 100644
--- a/integration/spark/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
+++ b/integration/spark/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
@@ -22,7 +22,7 @@ import java.io.IOException;
import java.sql.Date;
import java.sql.Timestamp;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index d73b782..0fecaaf 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -30,36 +30,33 @@ import org.apache.hadoop.conf.{Configurable, Configuration}
import org.apache.hadoop.fs.Path
import org.apache.hadoop.mapreduce.Job
import org.apache.hadoop.mapreduce.lib.input.{FileInputFormat, FileSplit}
-import org.apache.spark.{SparkContext, SparkEnv, SparkException}
import org.apache.spark.rdd.{DataLoadCoalescedRDD, DataLoadPartitionCoalescer, UpdateCoalescedRDD}
-import org.apache.spark.sql.{CarbonEnv, DataFrame, Row, SQLContext}
-import org.apache.spark.sql.execution.command.{AlterTableModel, CompactionCallableModel, CompactionModel, ExecutionErrors, UpdateTableModel}
+import org.apache.spark.sql.execution.command.{AlterTableModel, CompactionModel, ExecutionErrors, UpdateTableModel}
import org.apache.spark.sql.hive.DistributionUtil
+import org.apache.spark.sql.{CarbonEnv, DataFrame, Row, SQLContext}
import org.apache.spark.util.SparkUtil
+import org.apache.spark.{SparkEnv, SparkException}
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.{CarbonTableIdentifier, ColumnarFormatVersion}
-import org.apache.carbondata.core.datastore.block.{Distributable, TableBlockInfo}
-import org.apache.carbondata.core.metadata.CarbonMetadata
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.path.CarbonStorePath
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datastore.block.{Distributable, TableBlockInfo}
import org.apache.carbondata.core.dictionary.server.DictionaryServer
import org.apache.carbondata.core.load.{BlockDetails, LoadMetadataDetails}
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.core.metadata.{CarbonTableIdentifier, ColumnarFormatVersion}
+import org.apache.carbondata.core.path.CarbonStorePath
import org.apache.carbondata.core.update.CarbonUpdateUtil
-import org.apache.carbondata.core.updatestatus.SegmentStatusManager
+import org.apache.carbondata.core.updatestatus.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
import org.apache.carbondata.processing.csvreaderstep.RddInpututilsForUpdate
import org.apache.carbondata.processing.etl.DataLoadingException
import org.apache.carbondata.processing.model.CarbonLoadModel
import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException
import org.apache.carbondata.spark._
import org.apache.carbondata.spark.load._
-import org.apache.carbondata.spark.merger.{CarbonCompactionUtil, CarbonDataMergerUtil, CompactionCallable, CompactionType}
-import org.apache.carbondata.spark.partition.api.Partition
+import org.apache.carbondata.spark.merger.{CarbonCompactionUtil, CarbonDataMergerUtil, CompactionType}
import org.apache.carbondata.spark.splits.TableSplit
-import org.apache.carbondata.spark.util.{CarbonQueryUtil, CommonUtil, LoadMetadataUtil}
+import org.apache.carbondata.spark.util.{CarbonQueryUtil, CommonUtil}
/**
* This is the factory class which can create different RDD depends on user needs.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala
index b6fa827..2ca3b8c 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala
@@ -25,7 +25,7 @@ import org.apache.hadoop.mapred.JobConf
import org.apache.hadoop.mapreduce.Job
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat
-import org.apache.carbondata.core.AbsoluteTableIdentifier
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.hadoop.CarbonInputFormat
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
index 0ee44e1..eceb3a4 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
@@ -35,7 +35,7 @@ import org.apache.spark.sql.sources.{Filter, HadoopFsRelation, OutputWriterFacto
import org.apache.spark.sql.types.StructType
import org.apache.spark.util.SerializableConfiguration
-import org.apache.carbondata.core.AbsoluteTableIdentifier
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.path.CarbonTablePath
import org.apache.carbondata.hadoop.{CarbonInputFormat, CarbonInputSplit, CarbonProjection}
import org.apache.carbondata.hadoop.util.{CarbonInputFormatUtil, SchemaReader}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
index a779c67..476c74c 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
@@ -27,12 +27,10 @@ import org.apache.spark.sql.hive.{CarbonMetastore, CarbonMetastoreTypes}
import org.apache.spark.sql.optimizer.CarbonDecoderRelation
import org.apache.spark.sql.types._
-import org.apache.carbondata.core.{AbsoluteTableIdentifier, ColumnIdentifier}
import org.apache.carbondata.core.cache.{Cache, CacheProvider, CacheType}
import org.apache.carbondata.core.cache.dictionary.{Dictionary, DictionaryColumnUniqueIdentifier}
-import org.apache.carbondata.core.metadata.DataType
-import org.apache.carbondata.core.metadata.Encoding
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, ColumnIdentifier, DataType, Encoding}
import org.apache.carbondata.core.stats.{QueryStatistic, QueryStatisticsConstants}
import org.apache.carbondata.core.util.{CarbonTimeStatisticsFactory, DataTypeUtil}
import org.apache.carbondata.spark.CarbonAliasDecoderRelation
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
index b9bbe54..a0c3831 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/IUDCommands.scala
@@ -34,17 +34,17 @@ import org.apache.spark.storage.StorageLevel
import org.apache.spark.util.FileUtils
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.AbsoluteTableIdentifier
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.path.{CarbonStorePath, CarbonTablePath}
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastorage.impl.FileFactory
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.update._
import org.apache.carbondata.core.update.data.RowCountDetailsVO
import org.apache.carbondata.core.updatestatus.{SegmentStatusManager, SegmentUpdateStatusManager}
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.core.writer.CarbonDeleteDeltaWriterImpl
-import org.apache.carbondata.locks.{CarbonLockFactory, CarbonLockUtil, LockUsage}
+import org.apache.carbondata.core.updatestatus.locks.{CarbonLockFactory, CarbonLockUtil, LockUsage}
import org.apache.carbondata.processing.exception.MultipleMatchingException
import org.apache.carbondata.spark.load.{CarbonLoaderUtil, FailureCauses}
import org.apache.carbondata.spark.merger.{CarbonDataMergerUtil, CarbonDataMergerUtilResult, CompactionType}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index b25731e..93fb660 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -40,8 +40,7 @@ import org.codehaus.jackson.map.ObjectMapper
import org.apache.carbondata.api.CarbonStore
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.{CarbonDataLoadSchema, CarbonTableIdentifier}
-import org.apache.carbondata.core.metadata.Encoding
+import org.apache.carbondata.core.metadata.{CarbonTableIdentifier, Encoding}
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.path.CarbonStorePath
@@ -51,10 +50,10 @@ import org.apache.carbondata.core.dictionary.server.DictionaryServer
import org.apache.carbondata.core.update.CarbonUpdateUtil
import org.apache.carbondata.core.update.TupleIdEnum
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
-import org.apache.carbondata.locks.{CarbonLockFactory, LockUsage}
+import org.apache.carbondata.core.updatestatus.locks.{CarbonLockFactory, LockUsage}
import org.apache.carbondata.processing.constants.TableOptionConstant
import org.apache.carbondata.processing.etl.DataLoadingException
-import org.apache.carbondata.processing.model.CarbonLoadModel
+import org.apache.carbondata.processing.model.{CarbonDataLoadSchema, CarbonLoadModel}
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
import org.apache.carbondata.spark.rdd.{CarbonDataRDDFactory, DataManagementFunc, DictionaryLoadModel}
import org.apache.carbondata.spark.util.{CarbonScalaUtil, CommonUtil, GlobalDictionaryUtil}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
index 4c6c5be..123bef9 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
@@ -34,11 +34,10 @@ import org.apache.spark.sql.hive.client.ClientInterface
import org.apache.spark.sql.types._
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.CarbonTableIdentifier
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastorage.impl.FileFactory
import org.apache.carbondata.core.datastorage.impl.FileFactory.FileType
-import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.metadata.{CarbonMetadata, CarbonTableIdentifier}
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.path.{CarbonStorePath, CarbonTablePath}
@@ -47,7 +46,7 @@ import org.apache.carbondata.core.stats.{QueryStatistic, QueryStatisticsConstant
import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory, CarbonUtil}
import org.apache.carbondata.core.writer.ThriftWriter
import org.apache.carbondata.format.{SchemaEvolutionEntry, TableInfo}
-import org.apache.carbondata.locks.ZookeeperInit
+import org.apache.carbondata.core.updatestatus.locks.ZookeeperInit
import org.apache.carbondata.spark.merger.TableMeta
case class MetaData(var tablesMeta: ArrayBuffer[TableMeta])
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSystemLockFeatureTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSystemLockFeatureTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSystemLockFeatureTest.scala
index 21a627e..52999db 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSystemLockFeatureTest.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSystemLockFeatureTest.scala
@@ -23,10 +23,10 @@ import scala.collection.JavaConverters._
import org.apache.spark.sql.common.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.core.path.{CarbonStorePath, CarbonTablePath}
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastorage.impl.FileFactory
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.core.updatestatus.SegmentStatusManager
import org.apache.carbondata.core.util.CarbonProperties
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionMinorThresholdTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionMinorThresholdTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionMinorThresholdTest.scala
index a320349..09cb53c 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionMinorThresholdTest.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionMinorThresholdTest.scala
@@ -23,8 +23,8 @@ import scala.collection.JavaConverters._
import org.apache.spark.sql.common.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.core.updatestatus.SegmentStatusManager
import org.apache.carbondata.core.util.CarbonProperties
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionNoDictionaryTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionNoDictionaryTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionNoDictionaryTest.scala
index 8c3bcfc..c1203d7 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionNoDictionaryTest.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionNoDictionaryTest.scala
@@ -24,8 +24,8 @@ import org.apache.spark.sql.Row
import org.apache.spark.sql.common.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.core.updatestatus.SegmentStatusManager
import org.apache.carbondata.core.util.CarbonProperties
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala
index 3ccafce..e43fd4e 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala
@@ -24,14 +24,14 @@ import org.apache.spark.sql.Row
import org.apache.spark.sql.common.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.core.updatestatus.SegmentStatusManager
import org.apache.carbondata.core.util.CarbonProperties
/**
- * FT for data compaction scenario.
- */
+ * FT for data compaction scenario.
+ */
class DataCompactionTest extends QueryTest with BeforeAndAfterAll {
override def beforeAll {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
index 6032c34..1beed08 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala
@@ -24,11 +24,10 @@ import org.apache.spark.sql.common.util.QueryTest
import org.apache.spark.sql.{CarbonEnv, CarbonRelation}
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.CarbonDataLoadSchema
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.processing.constants.TableOptionConstant
-import org.apache.carbondata.processing.model.CarbonLoadModel
+import org.apache.carbondata.processing.model.{CarbonDataLoadSchema, CarbonLoadModel}
/**
* Test Case for org.apache.carbondata.integration.spark.util.GlobalDictionaryUtil
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala
index eb162ea..68710af 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala
@@ -25,13 +25,12 @@ import org.apache.spark.sql.common.util.QueryTest
import org.apache.spark.sql.{CarbonEnv, CarbonRelation}
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.{CarbonDataLoadSchema, CarbonTableIdentifier}
-import org.apache.carbondata.core.metadata.Encoding
+import org.apache.carbondata.core.metadata.{CarbonTableIdentifier, Encoding}
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.path.CarbonStorePath
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
-import org.apache.carbondata.processing.model.CarbonLoadModel
+import org.apache.carbondata.processing.model.{CarbonDataLoadSchema, CarbonLoadModel}
/**
* Test Case for org.apache.carbondata.spark.util.GlobalDictionaryUtil
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/DictionaryTestCaseUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/DictionaryTestCaseUtil.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/DictionaryTestCaseUtil.scala
index bd00c7b..5567aed 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/DictionaryTestCaseUtil.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/DictionaryTestCaseUtil.scala
@@ -20,9 +20,9 @@ package org.apache.carbondata.spark.util
import org.apache.spark.sql.CarbonRelation
import org.apache.spark.sql.test.TestQueryExecutor
-import org.apache.carbondata.core.CarbonTableIdentifier
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier
import org.apache.carbondata.spark.load.CarbonLoaderUtil
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
index ae5b63a..361d183 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala
@@ -22,12 +22,11 @@ import org.apache.spark.sql.common.util.QueryTest
import org.apache.spark.sql.{CarbonEnv, CarbonRelation}
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.CarbonDataLoadSchema
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.processing.constants.TableOptionConstant
import org.apache.carbondata.processing.etl.DataLoadingException
-import org.apache.carbondata.processing.model.CarbonLoadModel
+import org.apache.carbondata.processing.model.{CarbonDataLoadSchema, CarbonLoadModel}
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
index 21c111d..295d700 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
@@ -27,13 +27,12 @@ import org.apache.spark.sql.common.util.QueryTest
import org.apache.spark.sql.{CarbonEnv, CarbonRelation}
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.CarbonDataLoadSchema
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastorage.impl.FileFactory
import org.apache.carbondata.core.service.impl.PathFactory
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.processing.constants.TableOptionConstant
-import org.apache.carbondata.processing.model.CarbonLoadModel
+import org.apache.carbondata.processing.model.{CarbonDataLoadSchema, CarbonLoadModel}
class GlobalDictionaryUtilConcurrentTestCase extends QueryTest with BeforeAndAfterAll {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala
index 65fb846..7481c32 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala
@@ -24,11 +24,10 @@ import org.apache.spark.sql.common.util.QueryTest
import org.apache.spark.sql.{CarbonEnv, CarbonRelation}
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.CarbonDataLoadSchema
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.processing.constants.TableOptionConstant
-import org.apache.carbondata.processing.model.CarbonLoadModel
+import org.apache.carbondata.processing.model.{CarbonDataLoadSchema, CarbonLoadModel}
/**
* Test Case for org.apache.carbondata.spark.util.GlobalDictionaryUtil
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java b/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
index f4bb50a..3649858 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
@@ -20,7 +20,7 @@ package org.apache.carbondata.spark.readsupport;
import java.io.IOException;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.hadoop.readsupport.impl.AbstractDictionaryDecodedReadSupport;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonSparkFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonSparkFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonSparkFactory.scala
index 4b409ed..8922945 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonSparkFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/CarbonSparkFactory.scala
@@ -16,16 +16,16 @@
*/
package org.apache.carbondata.spark
-import org.apache.carbondata.core.{CarbonTableIdentifier, ColumnIdentifier}
+import org.apache.carbondata.core.metadata.{CarbonTableIdentifier, ColumnIdentifier}
import org.apache.carbondata.core.metadata.schema.table.column.{CarbonDimension, ColumnSchema}
-
- /**
- * Column validator
- */
+/**
+ * Column validator
+ */
trait ColumnValidator {
def validateColumns(columns: Seq[ColumnSchema])
}
+
/**
* Dictionary related helper service
*/
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark2/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
index 979e154..087a62c 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
@@ -20,9 +20,9 @@ import scala.collection.mutable.HashMap
import org.apache.carbondata.core.datastorage.filesystem.{CarbonFile, CarbonFileFilter}
import org.apache.carbondata.core.datastorage.impl.FileFactory
+import org.apache.carbondata.core.metadata.{CarbonTableIdentifier, ColumnIdentifier}
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.path.{CarbonStorePath, CarbonTablePath}
-import org.apache.carbondata.core.{CarbonTableIdentifier, ColumnIdentifier}
class DictionaryDetailHelper extends DictionaryDetailService {
def getDictionaryDetail(dictfolderPath: String, primDimensions: Array[CarbonDimension],
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index f687589..441500b 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -30,36 +30,32 @@ import org.apache.hadoop.conf.{Configurable, Configuration}
import org.apache.hadoop.fs.Path
import org.apache.hadoop.mapreduce.Job
import org.apache.hadoop.mapreduce.lib.input.{FileInputFormat, FileSplit}
-import org.apache.spark.{SparkContext, SparkEnv, SparkException}
import org.apache.spark.rdd.{DataLoadCoalescedRDD, DataLoadPartitionCoalescer, UpdateCoalescedRDD}
-import org.apache.spark.sql.{CarbonEnv, DataFrame, Row, SQLContext}
-import org.apache.spark.sql.execution.command.{AlterTableModel, CompactionCallableModel, CompactionModel, ExecutionErrors, UpdateTableModel}
+import org.apache.spark.sql.execution.command.{AlterTableModel, CompactionModel, ExecutionErrors, UpdateTableModel}
import org.apache.spark.sql.hive.DistributionUtil
+import org.apache.spark.sql.{CarbonEnv, DataFrame, Row, SQLContext}
import org.apache.spark.util.SparkUtil
+import org.apache.spark.{SparkEnv, SparkException}
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.{CarbonDataLoadSchema, CarbonTableIdentifier, ColumnarFormatVersion}
+import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastore.block.{Distributable, TableBlockInfo}
-import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.load.{BlockDetails, LoadMetadataDetails}
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.core.metadata.{CarbonTableIdentifier, ColumnarFormatVersion}
import org.apache.carbondata.core.path.CarbonStorePath
-import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.dictionary.server.DictionaryServer
-import org.apache.carbondata.core.load.{BlockDetails, LoadMetadataDetails}
import org.apache.carbondata.core.update.CarbonUpdateUtil
-import org.apache.carbondata.core.updatestatus.SegmentStatusManager
+import org.apache.carbondata.core.updatestatus.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
import org.apache.carbondata.processing.csvreaderstep.RddInpututilsForUpdate
import org.apache.carbondata.processing.etl.DataLoadingException
import org.apache.carbondata.processing.model.CarbonLoadModel
import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException
import org.apache.carbondata.spark._
import org.apache.carbondata.spark.load._
-import org.apache.carbondata.spark.merger.{CarbonCompactionUtil, CarbonDataMergerUtil, CompactionCallable, CompactionType}
-import org.apache.carbondata.spark.partition.api.Partition
+import org.apache.carbondata.spark.merger.{CarbonCompactionUtil, CarbonDataMergerUtil, CompactionType}
import org.apache.carbondata.spark.splits.TableSplit
-import org.apache.carbondata.spark.util.{CarbonQueryUtil, CommonUtil, LoadMetadataUtil}
+import org.apache.carbondata.spark.util.{CarbonQueryUtil, CommonUtil}
/**
* This is the factory class which can create different RDD depends on user needs.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala
index b6fa827..2ca3b8c 100644
--- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/util/QueryPlanUtil.scala
@@ -25,7 +25,7 @@ import org.apache.hadoop.mapred.JobConf
import org.apache.hadoop.mapreduce.Job
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat
-import org.apache.carbondata.core.AbsoluteTableIdentifier
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.hadoop.CarbonInputFormat
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
index f5c99a5..7633b22 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
@@ -17,9 +17,6 @@
package org.apache.spark.sql
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.mapred.JobConf
-import org.apache.hadoop.mapreduce.Job
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.execution.command.LoadTableByInsert
@@ -27,16 +24,15 @@ import org.apache.spark.sql.hive.CarbonRelation
import org.apache.spark.sql.sources.{BaseRelation, Filter, InsertableRelation}
import org.apache.spark.sql.types.StructType
-import org.apache.carbondata.core.AbsoluteTableIdentifier
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.hadoop.{CarbonInputFormat, CarbonProjection}
-import org.apache.carbondata.hadoop.util.SchemaReader
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
import org.apache.carbondata.core.scan.expression.Expression
import org.apache.carbondata.core.scan.expression.logical.AndExpression
+import org.apache.carbondata.hadoop.CarbonProjection
+import org.apache.carbondata.hadoop.util.SchemaReader
import org.apache.carbondata.spark.CarbonFilters
import org.apache.carbondata.spark.merger.TableMeta
import org.apache.carbondata.spark.rdd.CarbonScanRDD
-import org.apache.carbondata.spark.readsupport.SparkRowReadSupportImpl
import org.apache.carbondata.spark.util.CarbonSparkUtil
case class CarbonDatasourceHadoopRelation(
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
index f8c60a0..cb71a46 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
@@ -30,7 +30,7 @@ import org.apache.spark.sql.types._
import org.apache.carbondata.core.cache.{Cache, CacheProvider, CacheType}
import org.apache.carbondata.core.cache.dictionary.{Dictionary, DictionaryColumnUniqueIdentifier}
-import org.apache.carbondata.core.{AbsoluteTableIdentifier, ColumnIdentifier}
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, ColumnIdentifier}
import org.apache.carbondata.core.metadata.DataType
import org.apache.carbondata.core.metadata.Encoding
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index 4894e47..e662f96 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -18,7 +18,6 @@
package org.apache.spark.sql.execution.command
import java.io.File
-import java.text.SimpleDateFormat
import scala.collection.JavaConverters._
import scala.language.implicitConversions
@@ -27,33 +26,29 @@ import org.apache.commons.lang3.StringUtils
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
-import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Cast, Literal}
+import org.apache.spark.sql.catalyst.expressions.Attribute
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.SparkPlan
import org.apache.spark.sql.hive.{CarbonMetastore, CarbonRelation}
-import org.apache.spark.sql.types.TimestampType
import org.apache.spark.util.FileUtils
import org.codehaus.jackson.map.ObjectMapper
import org.apache.carbondata.api.CarbonStore
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.{CarbonDataLoadSchema, CarbonTableIdentifier}
-import org.apache.carbondata.core.metadata.CarbonMetadata
-import org.apache.carbondata.core.metadata.Encoding
-import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
-import org.apache.carbondata.core.path.CarbonStorePath
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastorage.impl.FileFactory
-import org.apache.carbondata.core.update.CarbonUpdateUtil
-import org.apache.carbondata.core.update.TupleIdEnum
+import org.apache.carbondata.core.metadata.{CarbonMetadata, CarbonTableIdentifier, Encoding}
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
+import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo}
+import org.apache.carbondata.core.path.CarbonStorePath
+import org.apache.carbondata.core.update.{CarbonUpdateUtil, TupleIdEnum}
+import org.apache.carbondata.core.updatestatus.locks.{CarbonLockFactory, LockUsage}
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
-import org.apache.carbondata.locks.{CarbonLockFactory, LockUsage}
import org.apache.carbondata.processing.constants.TableOptionConstant
import org.apache.carbondata.processing.etl.DataLoadingException
-import org.apache.carbondata.processing.model.CarbonLoadModel
+import org.apache.carbondata.processing.model.{CarbonDataLoadSchema, CarbonLoadModel}
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
-import org.apache.carbondata.spark.rdd.{CarbonDataRDDFactory, DataManagementFunc, DictionaryLoadModel}
+import org.apache.carbondata.spark.rdd.{CarbonDataRDDFactory, DictionaryLoadModel}
import org.apache.carbondata.spark.util.{CarbonScalaUtil, CarbonSparkUtil, CommonUtil, GlobalDictionaryUtil}
object Checker {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
index 4faa8a9..f299de9 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
@@ -35,7 +35,6 @@ import org.apache.spark.sql.types._
import org.apache.spark.sql.{RuntimeConfig, SparkSession}
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.CarbonTableIdentifier
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastorage.filesystem.CarbonFile
import org.apache.carbondata.core.datastorage.impl.FileFactory
@@ -43,15 +42,15 @@ import org.apache.carbondata.core.datastorage.impl.FileFactory.FileType
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
-import org.apache.carbondata.core.metadata.{CarbonMetadata, DataType => CarbonDataType}
+import org.apache.carbondata.core.metadata.{CarbonMetadata, CarbonTableIdentifier, DataType => CarbonDataType}
import org.apache.carbondata.core.path.{CarbonStorePath, CarbonTablePath}
import org.apache.carbondata.core.reader.ThriftReader
import org.apache.carbondata.core.stats.{QueryStatistic, QueryStatisticsConstants}
import org.apache.carbondata.core.updatestatus.SegmentStatusManager
+import org.apache.carbondata.core.updatestatus.locks.ZookeeperInit
import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory, CarbonUtil}
import org.apache.carbondata.core.writer.ThriftWriter
import org.apache.carbondata.format.{SchemaEvolutionEntry, TableInfo}
-import org.apache.carbondata.locks.ZookeeperInit
import org.apache.carbondata.spark.merger.TableMeta
import org.apache.carbondata.spark.util.CarbonSparkUtil
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java b/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
index 1eddc32..3599256 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvload/DataGraphExecuter.java
@@ -27,7 +27,7 @@ import java.util.Set;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.CarbonDataLoadSchema;
+import org.apache.carbondata.processing.model.CarbonDataLoadSchema;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/processing/src/main/java/org/apache/carbondata/processing/csvload/GraphExecutionUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvload/GraphExecutionUtil.java b/processing/src/main/java/org/apache/carbondata/processing/csvload/GraphExecutionUtil.java
index fbaf622..d136575 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/csvload/GraphExecutionUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvload/GraphExecutionUtil.java
@@ -32,8 +32,8 @@ import java.util.Set;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.CarbonDataLoadSchema;
-import org.apache.carbondata.core.CarbonDataLoadSchema.DimensionRelation;
+import org.apache.carbondata.processing.model.CarbonDataLoadSchema;
+import org.apache.carbondata.processing.model.CarbonDataLoadSchema.DimensionRelation;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java b/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java
index baa06bd..7abe631 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/datatypes/PrimitiveDataType.java
@@ -30,7 +30,7 @@ import java.util.Map;
import org.apache.carbondata.core.cache.Cache;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
-import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java b/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java
index 98bc8de..676f702 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java
@@ -37,7 +37,7 @@ import java.util.Set;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.CarbonDataLoadSchema;
+import org.apache.carbondata.processing.model.CarbonDataLoadSchema;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/processing/src/main/java/org/apache/carbondata/processing/model/CarbonDataLoadSchema.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/model/CarbonDataLoadSchema.java b/processing/src/main/java/org/apache/carbondata/processing/model/CarbonDataLoadSchema.java
new file mode 100644
index 0000000..f5db85f
--- /dev/null
+++ b/processing/src/main/java/org/apache/carbondata/processing/model/CarbonDataLoadSchema.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.processing.model;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+
+/**
+ * Wrapper Data Load Schema object which will be used to
+ * support relation while data loading
+ */
+public class CarbonDataLoadSchema implements Serializable {
+
+ /**
+ * default serializer
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * CarbonTable info
+ */
+ private CarbonTable carbonTable;
+
+ /**
+ * dimension table and relation info
+ */
+ private List<DimensionRelation> dimensionRelationList;
+
+ /**
+ * CarbonDataLoadSchema constructor which takes CarbonTable
+ *
+ * @param carbonTable
+ */
+ public CarbonDataLoadSchema(CarbonTable carbonTable) {
+ this.carbonTable = carbonTable;
+ this.dimensionRelationList = new ArrayList<DimensionRelation>();
+ }
+
+ /**
+ * get dimension relation list
+ *
+ * @return dimensionRelationList
+ */
+ public List<DimensionRelation> getDimensionRelationList() {
+ return dimensionRelationList;
+ }
+
+ /**
+ * get carbontable
+ *
+ * @return carbonTable
+ */
+ public CarbonTable getCarbonTable() {
+ return carbonTable;
+ }
+
+ /**
+ * Dimension Relation object which will be filled from
+ * Load DML Command to support normalized table data load
+ */
+ public static class DimensionRelation implements Serializable {
+ /**
+ * default serializer
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * dimension tableName
+ */
+ private String tableName;
+
+ /**
+ * relation with fact and dimension table
+ */
+ private Relation relation;
+
+ /**
+ * Columns to selected from dimension table.
+ * Hierarchy in-memory table should be prepared
+ * based on selected columns
+ */
+ private List<String> columns;
+
+ /**
+ * constructor
+ *
+ * @param tableName - dimension table name
+ * @param relation - fact foreign key with dimension primary key mapping
+ * @param columns - list of columns to be used from this dimension table
+ */
+ public DimensionRelation(String tableName, Relation relation,
+ List<String> columns) {
+ this.tableName = tableName;
+ this.relation = relation;
+ this.columns = columns;
+ }
+
+ /**
+ * @return tableName
+ */
+ public String getTableName() {
+ return tableName;
+ }
+
+ /**
+ * @return relation
+ */
+ public Relation getRelation() {
+ return relation;
+ }
+
+ /**
+ * @return columns
+ */
+ public List<String> getColumns() {
+ return columns;
+ }
+ }
+
+ /**
+ * Relation class to specify fact foreignkey column with
+ * dimension primary key column
+ */
+ public static class Relation implements Serializable {
+ /**
+ * default serializer
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Fact foreign key column
+ */
+ private String factForeignKeyColumn;
+
+ /**
+ * dimension primary key column
+ */
+ private String dimensionPrimaryKeyColumn;
+
+ /**
+ * constructor
+ *
+ * @param factForeignKeyColumn - Fact Table Foreign key
+ * @param dimensionPrimaryKeyColumn - Dimension Table primary key
+ */
+ public Relation(String factForeignKeyColumn, String dimensionPrimaryKeyColumn) {
+ this.factForeignKeyColumn = factForeignKeyColumn;
+ this.dimensionPrimaryKeyColumn = dimensionPrimaryKeyColumn;
+ }
+
+ /**
+ * @return factForeignKeyColumn
+ */
+ public String getFactForeignKeyColumn() {
+ return factForeignKeyColumn;
+ }
+
+ /**
+ * @return dimensionPrimaryKeyColumn
+ */
+ public String getDimensionPrimaryKeyColumn() {
+ return dimensionPrimaryKeyColumn;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java b/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
index 9696d36..8be315e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
@@ -26,7 +26,6 @@ import java.io.Serializable;
import java.util.HashMap;
import java.util.List;
-import org.apache.carbondata.core.CarbonDataLoadSchema;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.load.LoadMetadataDetails;
import org.apache.carbondata.core.update.SegmentUpdateDetails;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java
index 0d0bfc5..0eaee48 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java
@@ -20,7 +20,7 @@ package org.apache.carbondata.processing.newflow;
import java.util.HashMap;
import java.util.Map;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.BucketingInfo;
public class CarbonDataLoadConfiguration {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadExecutor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadExecutor.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadExecutor.java
index 5e7edbd..3092bd5 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadExecutor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadExecutor.java
@@ -19,7 +19,7 @@ package org.apache.carbondata.processing.newflow;
import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.processing.model.CarbonLoadModel;
import org.apache.carbondata.processing.newflow.exception.BadRecordFoundException;
import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
index 7ee40bc..8099427 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
@@ -24,7 +24,7 @@ import java.util.Map;
import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java
index 99f9e61..0a9e866 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/DictionaryFieldConverterImpl.java
@@ -27,7 +27,7 @@ import java.util.Map;
import org.apache.carbondata.core.cache.Cache;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
-import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.devapi.BiDictionary;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/FieldEncoderFactory.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/FieldEncoderFactory.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/FieldEncoderFactory.java
index ee36fd1..8dcff97 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/FieldEncoderFactory.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/converter/impl/FieldEncoderFactory.java
@@ -24,7 +24,7 @@ import java.util.List;
import org.apache.carbondata.core.cache.Cache;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
-import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
index 80e6128..ff906fe 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
@@ -24,7 +24,7 @@ import java.io.IOException;
import java.util.Iterator;
import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.processing.constants.LoggerAction;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
index e65846c..00fad6d 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
@@ -26,7 +26,7 @@ import java.util.Iterator;
import java.util.List;
import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.schema.BucketingInfo;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java
index ca261ab..131d7c0 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java
@@ -24,7 +24,7 @@ import java.util.Iterator;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.constants.IgnoreDictionary;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
index 8466aae..c6e76a4 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
@@ -22,7 +22,7 @@ import java.io.File;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.processing.newflow.CarbonDataLoadConfiguration;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataWriterFactory.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataWriterFactory.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataWriterFactory.java
index 3434ed8..b745bd2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataWriterFactory.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonDataWriterFactory.java
@@ -19,7 +19,7 @@
package org.apache.carbondata.processing.store;
-import org.apache.carbondata.core.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.processing.store.writer.CarbonDataWriterVo;
import org.apache.carbondata.processing.store.writer.CarbonFactDataWriter;
import org.apache.carbondata.processing.store.writer.v1.CarbonFactDataWriterImplV1;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
index 1de837c..c27190e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
@@ -41,7 +41,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index 8211839..801e023 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -25,7 +25,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
index a8a3aed..7b93d2c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
@@ -26,7 +26,7 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.BlockletInfoColumnar;
import org.apache.carbondata.core.util.CarbonMetadataUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
index 33e97e0..e7b9a4c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
@@ -33,8 +33,8 @@ import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.cache.CacheType;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index 326bde7..978ce56 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -36,7 +36,7 @@ import java.util.Set;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.CarbonDataLoadSchema;
+import org.apache.carbondata.processing.model.CarbonDataLoadSchema;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonSchemaParser.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonSchemaParser.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonSchemaParser.java
index 4480ada..9b58a84 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonSchemaParser.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonSchemaParser.java
@@ -30,8 +30,8 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-import org.apache.carbondata.core.CarbonDataLoadSchema;
-import org.apache.carbondata.core.CarbonDataLoadSchema.DimensionRelation;
+import org.apache.carbondata.processing.model.CarbonDataLoadSchema;
+import org.apache.carbondata.processing.model.CarbonDataLoadSchema.DimensionRelation;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java b/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java
index 3a87edc..d392f67 100644
--- a/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java
+++ b/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java
@@ -33,9 +33,9 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.cache.CacheType;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.datastore.BlockIndexStore;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
[04/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scan/expression/conditional/GreaterThanExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/expression/conditional/GreaterThanExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/scan/expression/conditional/GreaterThanExpressionUnitTest.java
deleted file mode 100644
index e0805a6..0000000
--- a/core/src/test/java/org/apache/carbondata/scan/expression/conditional/GreaterThanExpressionUnitTest.java
+++ /dev/null
@@ -1,364 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.expression.conditional;
-
-import java.math.BigDecimal;
-import java.sql.Timestamp;
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.intf.RowImpl;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.apache.spark.sql.types.Decimal;
-import org.junit.Test;
-
-import static junit.framework.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-public class GreaterThanExpressionUnitTest {
- static GreaterThanExpression greaterThanExpression;
-
- @Test public void testEvaluateForGreaterThanExpressionWithStringDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
- right.setColIndex(1);
- greaterThanExpression = new GreaterThanExpression(left, right);
- RowImpl value = new RowImpl();
- String[] row = { "string1" };
- String[] row1 = { "String's Value" };
- Object objectRow[] = { row, row1 };
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public String getString() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return "string1";
-
- } else {
- return "String's Value";
-
- }
-
- }
- };
- value.setValues(objectRow);
- ExpressionResult result = greaterThanExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForGreaterThanExpressionWithShortDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
- right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("id", DataType.SHORT);
- left.setColIndex(1);
- greaterThanExpression = new GreaterThanExpression(left, right);
- RowImpl value = new RowImpl();
- Short[] row = { 170 };
- Short[] row1 = { 70 };
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public Short getShort() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return 170;
-
- } else {
- return 70;
-
- }
-
- }
- };
-
- ExpressionResult result = greaterThanExpression.evaluate(value);
- assertTrue(result.getBoolean());
-
- }
-
- @Test public void testEvaluateForGreaterThanExpressionWithDoubleDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("right_contact", DataType.DOUBLE);
- right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("left_contact", DataType.DOUBLE);
- left.setColIndex(1);
- greaterThanExpression = new GreaterThanExpression(left, right);
- RowImpl value = new RowImpl();
- Double[] row = { 44D };
- Double[] row1 = { 20D };
- Object objectRow[] = { row1, row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public Double getDouble() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return 44D;
-
- } else {
- return 20D;
-
- }
-
- }
- };
-
- ExpressionResult result = greaterThanExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForGreaterThanExpressionWithIntDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("right_number", DataType.INT);
- right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("left_number", DataType.INT);
- left.setColIndex(1);
- greaterThanExpression = new GreaterThanExpression(left, right);
- RowImpl value = new RowImpl();
- Integer[] row = { 140 };
- Integer[] row1 = { 150 };
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public Integer getInt() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return 150;
-
- } else {
- return 140;
-
- }
-
- }
- };
-
- ExpressionResult result = greaterThanExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForGreaterThanExpressionWithTimestampDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- try {
- ColumnExpression left = new ColumnExpression("timestamp", DataType.TIMESTAMP);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("timestamp", DataType.TIMESTAMP);
- right.setColIndex(1);
-
- greaterThanExpression = new GreaterThanExpression(left, right);
-
- RowImpl value = new RowImpl();
-
- DateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy");
-
- Date date = dateFormat.parse("23/09/2007");
- long time = date.getTime();
- Timestamp[] row = { new Timestamp(time) };
-
- Date date1 = dateFormat.parse("24/09/2007");
- long time1 = date1.getTime();
- Timestamp[] row1 = { new Timestamp(time1) };
-
- Object objectRow[] = { row1, row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public Long getTime() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return 1190592000L;
- } else {
- return 1190505600L;
- }
- }
- };
-
- ExpressionResult result = greaterThanExpression.evaluate(value);
- assertTrue(result.getBoolean());
- } catch (ParseException e) {
- System.out.println("Error while parsing " + e.getMessage());
- }
- }
-
- @Test public void testEvaluateForGreaterThanExpressionWithLongDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("contact", DataType.LONG);
- right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("contact", DataType.LONG);
- left.setColIndex(1);
- greaterThanExpression = new GreaterThanExpression(left, right);
- RowImpl value = new RowImpl();
- Long[] row = { 1234567654321L };
- Long[] row1 = { 123456765432234L };
- Object objectRow[] = { row1, row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public Long getLong() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return 123456765432234L;
- } else {
- return 1234567654321L;
- }
- }
- };
-
- ExpressionResult result = greaterThanExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForGreaterThanExpressionWithDecimalDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("contact", DataType.DECIMAL);
- right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("contact", DataType.DECIMAL);
- left.setColIndex(1);
- greaterThanExpression = new GreaterThanExpression(left, right);
- RowImpl value = new RowImpl();
- Decimal[] row = new Decimal[] { Decimal.apply(12345.0) };
- Decimal[] row1 = new Decimal[] { Decimal.apply(123451245.0) };
- Object objectRow[] = { row1, row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public BigDecimal getDecimal() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return new BigDecimal(123451245.0);
- } else {
- return new BigDecimal(12345.0);
- }
- }
- };
-
- ExpressionResult result = greaterThanExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test(expected = FilterUnsupportedException.class) public void testForGreaterThanExpressionWithDefaultCase()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("contact", DataType.BOOLEAN);
- right.setColIndex(0);
- greaterThanExpression = new GreaterThanExpression(right, right);
- RowImpl value = new RowImpl();
- Boolean[] row = { true };
- Object objectRow[] = { row };
- value.setValues(objectRow);
- greaterThanExpression.evaluate(value);
- }
-
- @Test public void testEvaluateForGreaterThanExpressionWithIsNullReturnTrue()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
- right.setColIndex(0);
- greaterThanExpression = new GreaterThanExpression(right, right);
- RowImpl value = new RowImpl();
- Short[] row = { 15 };
- Object objectRow[] = { row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public boolean isNull() {
- return true;
- }
- };
-
- new MockUp<ExpressionResult>() {
- @Mock public Short getShort() {
- return 15;
- }
- };
-
- ExpressionResult result = greaterThanExpression.evaluate(value);
- assertFalse(result.getBoolean());
-
- }
-
- @Test public void testEvaluateForGreaterThanExpressionWithLeftAndRightDifferentDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression left = new ColumnExpression("name", DataType.STRING);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("number", DataType.INT);
- right.setColIndex(1);
- greaterThanExpression = new GreaterThanExpression(left, right);
- RowImpl value = new RowImpl();
- String[] row = { "String1" };
- Integer[] row1 = { 14 };
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public Integer getInt() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return 15;
- } else {
- return 14;
-
- }
-
- }
- };
-
- ExpressionResult result = greaterThanExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testForGreaterThanExpressionWithGetString() throws Exception {
- ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
- right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
- left.setColIndex(0);
- greaterThanExpression = new GreaterThanExpression(left, right);
- String expected_result = "GreaterThan(ColumnExpression(left_name),ColumnExpression(right_name))";
- String result = greaterThanExpression.getString();
- assertEquals(expected_result, result);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scan/expression/conditional/InExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/expression/conditional/InExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/scan/expression/conditional/InExpressionUnitTest.java
deleted file mode 100644
index fd7bde9..0000000
--- a/core/src/test/java/org/apache/carbondata/scan/expression/conditional/InExpressionUnitTest.java
+++ /dev/null
@@ -1,276 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.expression.conditional;
-
-import java.math.BigDecimal;
-import java.sql.Timestamp;
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.intf.RowImpl;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.apache.spark.sql.types.Decimal;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-public class InExpressionUnitTest {
-
- static InExpression inExpression;
-
- @Test public void testEvaluateForInExpressionWithString()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
- right.setColIndex(1);
- inExpression = new InExpression(left, right);
- RowImpl value = new RowImpl();
- String row = "string1";
- String row1 = "string1";
- Object objectRow[] = { row, row1 };
-
- new MockUp<ExpressionResult>() {
-
- @Mock public DataType getDataType() {
- return DataType.STRING;
- }
-
- @Mock public String getString() {
- return "string1";
- }
- };
-
- value.setValues(objectRow);
- ExpressionResult result = inExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForInExpressionWithShortDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
-
- ColumnExpression left = new ColumnExpression("left_id", DataType.SHORT);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("right_id", DataType.SHORT);
- right.setColIndex(1);
- inExpression = new InExpression(left, right);
- RowImpl value = new RowImpl();
- Short row = 150;
- Short row1 = 150;
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public Short getShort() {
- return 150;
- }
- };
-
- ExpressionResult result = inExpression.evaluate(value);
- assertEquals(result.getDataType(), DataType.BOOLEAN);
-
- }
-
- @Test public void testEvaluateForInExpressionWithIntDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
-
- ColumnExpression left = new ColumnExpression("left_id", DataType.INT);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("right_id", DataType.INT);
- right.setColIndex(1);
- inExpression = new InExpression(left, right);
- RowImpl value = new RowImpl();
- Integer row = 15052;
- Integer row1 = 15052;
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public Integer getInt() {
- return 15052;
- }
- };
-
- ExpressionResult result = inExpression.evaluate(value);
- assertEquals(result.getDataType(), DataType.BOOLEAN);
-
- }
-
- @Test public void testEvaluateForInExpressionWithDoubleDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression left = new ColumnExpression("left_contact", DataType.DOUBLE);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("right_contact", DataType.DOUBLE);
- right.setColIndex(1);
- inExpression = new InExpression(left, right);
- RowImpl value = new RowImpl();
- Double row = 44521D;
- Double row1 = 44521D;
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public Double getDouble() {
- return 44521D;
- }
- };
-
- ExpressionResult result = inExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForInExpressionWithLongDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression left = new ColumnExpression("left_contact", DataType.LONG);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("right_contact", DataType.LONG);
- right.setColIndex(1);
- inExpression = new InExpression(left, right);
- RowImpl value = new RowImpl();
- Long row = 1234567654321L;
- Long row1 = 1234567654321L;
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public Long getLong() {
- return 1234567654321L;
- }
- };
-
- ExpressionResult result = inExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForInExpressionWithTimestampDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- try {
- ColumnExpression left = new ColumnExpression("left_timestamp", DataType.TIMESTAMP);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("right_timestamp", DataType.TIMESTAMP);
- right.setColIndex(1);
- inExpression = new InExpression(left, right);
-
- RowImpl value = new RowImpl();
- DateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy");
- Date date = dateFormat.parse("23/09/2007");
-
- long time = date.getTime();
-
- Timestamp row = new Timestamp(time);
- Timestamp row1 = new Timestamp(time);
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public Long getTime() {
- return 18465213000000L;
- }
- };
-
- ExpressionResult result = inExpression.evaluate(value);
- assertFalse(result.getBoolean());
- } catch (ParseException e) {
- System.out.println("Error while parsing " + e.getMessage());
- }
- }
-
- @Test public void testEvaluateForInExpressionWithDecimalDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression left = new ColumnExpression("left_contact", DataType.DECIMAL);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("right_contact", DataType.DECIMAL);
- right.setColIndex(1);
- inExpression = new InExpression(left, right);
- RowImpl value = new RowImpl();
- Decimal row = Decimal.apply(123452154.0);
- Decimal row1 = Decimal.apply(123452154.0);
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public BigDecimal getDecimal() {
- return new BigDecimal(123452154.0);
- }
- };
-
- ExpressionResult result = inExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test(expected = FilterUnsupportedException.class) public void testForInExpressionWithDefaultCase()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression left = new ColumnExpression("contact", DataType.BOOLEAN);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("contact", DataType.BOOLEAN);
- right.setColIndex(1);
- inExpression = new InExpression(left, right);
- RowImpl value = new RowImpl();
- Boolean row = true;
- Boolean row1 = true;
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
- inExpression.evaluate(value);
- }
-
- @Test public void testForInExpressionWithGetString() throws Exception {
- ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
- right.setColIndex(1);
- inExpression = new InExpression(left, right);
- String expected_result = "IN(ColumnExpression(left_name),ColumnExpression(right_name))";
- String result = inExpression.getString();
- assertEquals(expected_result, result);
- }
-
- @Test public void testEvaluateForInExpressionWithLeftAndRightDifferentDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("name", DataType.STRING);
- right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("number", DataType.INT);
- left.setColIndex(1);
- inExpression = new InExpression(left, right);
- RowImpl value = new RowImpl();
- String row1 = "String1";
- Integer row = 14523 ;
- Object objectRow[] = { row1, row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public Integer getInt() {
- return 14523;
- }
- };
-
- ExpressionResult result = inExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java
deleted file mode 100644
index f7aaa99..0000000
--- a/core/src/test/java/org/apache/carbondata/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java
+++ /dev/null
@@ -1,368 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.expression.conditional;
-
-import java.math.BigDecimal;
-import java.sql.Timestamp;
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.intf.RowImpl;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.apache.spark.sql.types.Decimal;
-import org.junit.Test;
-
-import static junit.framework.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-public class LessThanEqualToExpressionUnitTest {
-
- static LessThanEqualToExpression lessThanEqualToExpression;
-
- @Test public void testEvaluateForLessThanEqualToExpressionWithBothStringISSame()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
- left.setColIndex(1);
- lessThanEqualToExpression = new LessThanEqualToExpression(left, left);
- RowImpl value = new RowImpl();
- String[] row = { "String is Value" };
- String[] row1 = { "string1" };
- Object objectRow[] = { row, row1 };
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public String getString() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return "String is Value";
-
- } else {
- return "string1";
-
- }
-
- }
- };
-
- value.setValues(objectRow);
- ExpressionResult result = lessThanEqualToExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForLessThanEqualToExpressionWithShortDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
- right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("id", DataType.SHORT);
- left.setColIndex(1);
- lessThanEqualToExpression = new LessThanEqualToExpression(left, right);
- RowImpl value = new RowImpl();
- Short[] row = { 1550 };
- Short[] row1 = { 3365 };
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public Short getShort() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return 1550;
-
- } else {
- return 3365;
-
- }
-
- }
- };
-
- ExpressionResult result = lessThanEqualToExpression.evaluate(value);
- assertTrue(result.getBoolean());
-
- }
-
- @Test public void testEvaluateForLessThanEqualToExpressionWithDoubleDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("right_contact", DataType.DOUBLE);
- right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("left_contact", DataType.DOUBLE);
- left.setColIndex(1);
- lessThanEqualToExpression = new LessThanEqualToExpression(left, right);
- RowImpl value = new RowImpl();
- Double[] row = { 4852.2D };
- Double[] row1 = { 4852.2D };
- Object objectRow[] = { row1, row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public Double getDouble() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return 4852.2D;
-
- } else {
- return 4852.2D;
-
- }
-
- }
- };
-
- ExpressionResult result = lessThanEqualToExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForLessThanEqualToExpressionWithIntDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("right_number", DataType.INT);
- right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("left_number", DataType.INT);
- left.setColIndex(1);
- lessThanEqualToExpression = new LessThanEqualToExpression(left, right);
- RowImpl value = new RowImpl();
- Integer[] row = { 144580 };
- Integer[] row1 = { 14500 };
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public Integer getInt() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return 14500;
-
- } else {
- return 144580;
-
- }
-
- }
- };
-
- ExpressionResult result = lessThanEqualToExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForLessThanEqualToExpressionWithTimestampDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- try {
- ColumnExpression left = new ColumnExpression("timestamp", DataType.TIMESTAMP);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("timestamp", DataType.TIMESTAMP);
- right.setColIndex(1);
-
- lessThanEqualToExpression = new LessThanEqualToExpression(left, right);
-
- RowImpl value = new RowImpl();
-
- DateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy");
-
- Date date = dateFormat.parse("23/09/2007");
- long time = date.getTime();
- Timestamp[] row = { new Timestamp(time) };
-
- Date date1 = dateFormat.parse("24/09/2007");
- long time1 = date1.getTime();
- Timestamp[] row1 = { new Timestamp(time1) };
-
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public Long getTime() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return 1190505600L;
- } else {
- return 1190592000L;
- }
- }
- };
-
- ExpressionResult result = lessThanEqualToExpression.evaluate(value);
- assertTrue(result.getBoolean());
- } catch (ParseException e) {
- System.out.println("Error while parsing " + e.getMessage());
- }
- }
-
- @Test public void testEvaluateForLessThanEqualToExpressionWithLongDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("right_contact", DataType.LONG);
- right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("left_contact", DataType.LONG);
- left.setColIndex(1);
- lessThanEqualToExpression = new LessThanEqualToExpression(left, right);
- RowImpl value = new RowImpl();
- Long[] row = { 4751256L };
- Long[] row1 = { 48512586L };
- Object objectRow[] = { row1, row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public Long getLong() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return 4751256L;
- } else {
- return 48512586L;
- }
- }
- };
-
- ExpressionResult result = lessThanEqualToExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForLessThanEqualToExpressionWithDecimalDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("right_contact", DataType.DECIMAL);
- right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("left_contact", DataType.DECIMAL);
- left.setColIndex(1);
- lessThanEqualToExpression = new LessThanEqualToExpression(left, right);
- RowImpl value = new RowImpl();
- Decimal[] row = new Decimal[] { Decimal.apply(46851.2) };
- Decimal[] row1 = new Decimal[] { Decimal.apply(45821.02) };
- Object objectRow[] = { row1, row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public BigDecimal getDecimal() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return new BigDecimal(45821.02);
- } else {
- return new BigDecimal(46851.2);
- }
- }
- };
-
- ExpressionResult result = lessThanEqualToExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test(expected = FilterUnsupportedException.class) public void testForLessThanEqualToExpressionWithDefaultCase()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("contact", DataType.BOOLEAN);
- right.setColIndex(0);
- lessThanEqualToExpression = new LessThanEqualToExpression(right, right);
- RowImpl value = new RowImpl();
- Boolean[] row = { true };
- Object objectRow[] = { row };
- value.setValues(objectRow);
- lessThanEqualToExpression.evaluate(value);
- }
-
- @Test public void testEvaluateForLessThanEqualToExpressionWithIsNullReturnTrue()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
- right.setColIndex(0);
- lessThanEqualToExpression = new LessThanEqualToExpression(right, right);
- RowImpl value = new RowImpl();
- Short[] row = { 15856 };
- Object objectRow[] = { row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public boolean isNull() {
- return true;
- }
- };
-
- new MockUp<ExpressionResult>() {
- @Mock public Short getShort() {
- return 15856;
- }
- };
-
- ExpressionResult result = lessThanEqualToExpression.evaluate(value);
- assertFalse(result.getBoolean());
-
- }
-
- @Test public void testEvaluateForLessThanEqualToExpressionWithLeftAndRightDifferentDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression left = new ColumnExpression("name", DataType.STRING);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("number", DataType.INT);
- right.setColIndex(1);
- lessThanEqualToExpression = new LessThanEqualToExpression(left, right);
- RowImpl value = new RowImpl();
- String[] row = { "S" };
- Integer[] row1 = { 1450 };
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public Integer getInt() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return 84;
- } else {
- return 1450;
-
- }
-
- }
- };
-
- ExpressionResult result = lessThanEqualToExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testForLessThanEqualToExpressionWithGetString() throws Exception {
- ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
- right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
- left.setColIndex(0);
- lessThanEqualToExpression = new LessThanEqualToExpression(left, right);
- String expected_result =
- "LessThanEqualTo(ColumnExpression(left_name),ColumnExpression(right_name))";
- String result = lessThanEqualToExpression.getString();
- assertEquals(expected_result, result);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scan/expression/conditional/LessThanExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/expression/conditional/LessThanExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/scan/expression/conditional/LessThanExpressionUnitTest.java
deleted file mode 100644
index be3ab9c..0000000
--- a/core/src/test/java/org/apache/carbondata/scan/expression/conditional/LessThanExpressionUnitTest.java
+++ /dev/null
@@ -1,364 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.expression.conditional;
-
-import java.math.BigDecimal;
-import java.sql.Timestamp;
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.intf.RowImpl;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.apache.spark.sql.types.Decimal;
-import org.junit.Test;
-
-import static junit.framework.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-public class LessThanExpressionUnitTest {
- static LessThanExpression lessThanExpression;
-
- @Test public void testEvaluateForLessThanExpressionWithStringDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
- right.setColIndex(1);
- lessThanExpression = new LessThanExpression(left, right);
- RowImpl value = new RowImpl();
- String[] row = { "First String Value" };
- String[] row1 = { "string1" };
- Object objectRow[] = { row, row1 };
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public String getString() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return "First String Value";
-
- } else {
- return "string1";
-
- }
-
- }
- };
- value.setValues(objectRow);
- ExpressionResult result = lessThanExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForLessThanExpressionWithShortDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
- right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("id", DataType.SHORT);
- left.setColIndex(1);
- lessThanExpression = new LessThanExpression(left, right);
- RowImpl value = new RowImpl();
- Short[] row = { 7052 };
- Short[] row1 = { 7450 };
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public Short getShort() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return 7052;
-
- } else {
- return 7450;
-
- }
-
- }
- };
-
- ExpressionResult result = lessThanExpression.evaluate(value);
- assertTrue(result.getBoolean());
-
- }
-
- @Test public void testEvaluateForLessThanExpressionWithDoubleDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("right_contact", DataType.DOUBLE);
- right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("left_contact", DataType.DOUBLE);
- left.setColIndex(1);
- lessThanExpression = new LessThanExpression(left, right);
- RowImpl value = new RowImpl();
- Double[] row = { 2087D };
- Double[] row1 = { 4454D };
- Object objectRow[] = { row1, row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public Double getDouble() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return 2087D;
-
- } else {
- return 4454D;
-
- }
-
- }
- };
-
- ExpressionResult result = lessThanExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForLessThanExpressionWithIntDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("right_number", DataType.INT);
- right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("left_number", DataType.INT);
- left.setColIndex(1);
- lessThanExpression = new LessThanExpression(left, right);
- RowImpl value = new RowImpl();
- Integer[] row = { 1550 };
- Integer[] row1 = { 1420 };
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public Integer getInt() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return 1420;
-
- } else {
- return 1550;
-
- }
-
- }
- };
-
- ExpressionResult result = lessThanExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForLessThanExpressionWithTimestampDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- try {
- ColumnExpression left = new ColumnExpression("timestamp", DataType.TIMESTAMP);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("timestamp", DataType.TIMESTAMP);
- right.setColIndex(1);
-
- lessThanExpression = new LessThanExpression(left, right);
-
- RowImpl value = new RowImpl();
-
- DateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy");
-
- Date date = dateFormat.parse("23/09/2007");
- long time = date.getTime();
- Timestamp[] row = { new Timestamp(time) };
-
- Date date1 = dateFormat.parse("24/09/2007");
- long time1 = date1.getTime();
- Timestamp[] row1 = { new Timestamp(time1) };
-
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public Long getTime() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return 1190505600L;
- } else {
- return 1190592000L;
- }
- }
- };
-
- ExpressionResult result = lessThanExpression.evaluate(value);
- assertTrue(result.getBoolean());
- } catch (ParseException e) {
- System.out.println("Error while parsing " + e.getMessage());
- }
- }
-
- @Test public void testEvaluateForLessThanExpressionWithLongDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("contact", DataType.LONG);
- right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("contact", DataType.LONG);
- left.setColIndex(1);
- lessThanExpression = new LessThanExpression(left, right);
- RowImpl value = new RowImpl();
- Long[] row = { 14523656L };
- Long[] row1 = { 12456325L };
- Object objectRow[] = { row1, row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public Long getLong() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return 12456325L;
- } else {
- return 14523656L;
- }
- }
- };
-
- ExpressionResult result = lessThanExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForLessThanExpressionWithDecimalDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("contact", DataType.DECIMAL);
- right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("contact", DataType.DECIMAL);
- left.setColIndex(1);
- lessThanExpression = new LessThanExpression(left, right);
- RowImpl value = new RowImpl();
- Decimal[] row = new Decimal[] { Decimal.apply(256324.0) };
- Decimal[] row1 = new Decimal[] { Decimal.apply(123451245.0) };
- Object objectRow[] = { row1, row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public BigDecimal getDecimal() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return new BigDecimal(256324.0);
- } else {
- return new BigDecimal(123451245.0);
- }
- }
- };
-
- ExpressionResult result = lessThanExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test(expected = FilterUnsupportedException.class) public void testForLessThanExpressionWithDefaultCase()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("contact", DataType.BOOLEAN);
- right.setColIndex(0);
- lessThanExpression = new LessThanExpression(right, right);
- RowImpl value = new RowImpl();
- Boolean[] row = { true };
- Object objectRow[] = { row };
- value.setValues(objectRow);
- lessThanExpression.evaluate(value);
- }
-
- @Test public void testEvaluateForLessThanExpressionWithIsNullReturnTrue()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
- right.setColIndex(0);
- lessThanExpression = new LessThanExpression(right, right);
- RowImpl value = new RowImpl();
- Short[] row = { 15 };
- Object objectRow[] = { row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public boolean isNull() {
- return true;
- }
- };
-
- new MockUp<ExpressionResult>() {
- @Mock public Short getShort() {
- return 15;
- }
- };
-
- ExpressionResult result = lessThanExpression.evaluate(value);
- assertFalse(result.getBoolean());
-
- }
-
- @Test public void testEvaluateForLessThanExpressionWithLeftAndRightDifferentDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression left = new ColumnExpression("name", DataType.STRING);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("number", DataType.INT);
- right.setColIndex(1);
- lessThanExpression = new LessThanExpression(left, right);
- RowImpl value = new RowImpl();
- String[] row = { "S" };
- Integer[] row1 = { 1864 };
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public Integer getInt() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return 84;
- } else {
- return 1864;
-
- }
-
- }
- };
-
- ExpressionResult result = lessThanExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testForLessThanExpressionWithGetString() throws Exception {
- ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
- right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
- left.setColIndex(0);
- lessThanExpression = new LessThanExpression(left, right);
- String expected_result = "LessThan(ColumnExpression(left_name),ColumnExpression(right_name))";
- String result = lessThanExpression.getString();
- assertEquals(expected_result, result);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scan/expression/conditional/ListExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/expression/conditional/ListExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/scan/expression/conditional/ListExpressionUnitTest.java
deleted file mode 100644
index 9722339..0000000
--- a/core/src/test/java/org/apache/carbondata/scan/expression/conditional/ListExpressionUnitTest.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.expression.conditional;
-
-import java.util.List;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.intf.RowImpl;
-
-import org.junit.Test;
-
-import java.util.ArrayList;
-
-import static org.hamcrest.core.Is.is;
-import static org.hamcrest.core.IsEqual.equalTo;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-
-public class ListExpressionUnitTest {
-
- static ListExpression listExpression;
-
- @Test public void test() throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
- right.setColIndex(1);
-
- List<Expression> children = new ArrayList<>();
- children.add(left);
- children.add(right);
-
- listExpression = new ListExpression(children);
- RowImpl value = new RowImpl();
- String row = "Row is for left";
- String row1 = "I am row 1";
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
- String expected_value = "Row is for left";
- ExpressionResult result = listExpression.evaluate(value);
- assertThat(expected_value, is(equalTo(result.getList().get(0).getString())));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scan/expression/conditional/NotEqualsExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/expression/conditional/NotEqualsExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/scan/expression/conditional/NotEqualsExpressionUnitTest.java
deleted file mode 100644
index 8a1c814..0000000
--- a/core/src/test/java/org/apache/carbondata/scan/expression/conditional/NotEqualsExpressionUnitTest.java
+++ /dev/null
@@ -1,406 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.expression.conditional;
-
-import java.math.BigDecimal;
-import java.sql.Timestamp;
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.intf.RowImpl;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.apache.spark.sql.types.Decimal;
-import org.junit.Test;
-
-import static junit.framework.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-public class NotEqualsExpressionUnitTest {
-
- static NotEqualsExpression notEqualsExpression;
-
- @Test public void testEvaluateForNotEqualsExpressionWithBothStringISSame()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
- right.setColIndex(1);
- notEqualsExpression = new NotEqualsExpression(left, right);
- RowImpl value = new RowImpl();
- String[] row = { "string1" };
- String[] row1 = { "string2" };
- Object objectRow[] = { row, row1 };
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public String getString() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return "string1";
-
- } else {
- return "string2";
- }
- }
- };
-
- value.setValues(objectRow);
- ExpressionResult result = notEqualsExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForNotEqualsExpressionWithShortDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
-
- ColumnExpression left = new ColumnExpression("left_id", DataType.SHORT);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("right_id", DataType.SHORT);
- right.setColIndex(1);
- notEqualsExpression = new NotEqualsExpression(left, right);
- RowImpl value = new RowImpl();
- Short[] row = { 15 };
- Short[] row1 = { 16 };
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public Short getShort() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return 15;
-
- } else {
- return 16;
-
- }
-
- }
- };
-
- ExpressionResult result = notEqualsExpression.evaluate(value);
- assertTrue(result.getBoolean());
-
- }
-
- @Test public void testEvaluateForNotEqualsExpressionWithIntDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("right_number", DataType.INT);
- right.setColIndex(1);
- ColumnExpression left = new ColumnExpression("left_number", DataType.INT);
- left.setColIndex(0);
- notEqualsExpression = new NotEqualsExpression(left, right);
- RowImpl value = new RowImpl();
- Integer[] row = { 15 };
- Integer[] row1 = { 16 };
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public Integer getInt() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return 15;
-
- } else {
- return 16;
- }
- }
- };
-
- ExpressionResult result = notEqualsExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForNotEqualsExpressionWithDoubleDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("right_contact", DataType.DOUBLE);
- right.setColIndex(1);
- ColumnExpression left = new ColumnExpression("left_contact", DataType.DOUBLE);
- left.setColIndex(0);
- notEqualsExpression = new NotEqualsExpression(left, right);
- RowImpl value = new RowImpl();
- Double[] row = { 445.2D };
- Double[] row1 = { 452.08D };
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public Double getDouble() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return 445.2D;
-
- } else {
- return 452.08D;
- }
- }
- };
-
- ExpressionResult result = notEqualsExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForNotEqualsExpressionWithLongDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("right_contact", DataType.LONG);
- right.setColIndex(1);
- ColumnExpression left = new ColumnExpression("left_contact", DataType.LONG);
- left.setColIndex(0);
- notEqualsExpression = new NotEqualsExpression(left, right);
- RowImpl value = new RowImpl();
- Long[] row = { 1234567654321L };
- Long[] row1 = { 12345676541L };
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public Long getLong() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return 1234567654321L;
- } else {
- return 12345676541L;
- }
- }
- };
-
- ExpressionResult result = notEqualsExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForNotEqualsExpressionWithTimestampDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- try {
- ColumnExpression left = new ColumnExpression("left_timestamp", DataType.TIMESTAMP);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("right_timestamp", DataType.TIMESTAMP);
- right.setColIndex(1);
-
- notEqualsExpression = new NotEqualsExpression(left, right);
-
- RowImpl value = new RowImpl();
-
- DateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy");
-
- Date date = dateFormat.parse("23/09/2007");
- long time = date.getTime();
- Timestamp[] row = { new Timestamp(time) };
-
- Date date1 = dateFormat.parse("24/09/2007");
- long time1 = date1.getTime();
- Timestamp[] row1 = { new Timestamp(time1) };
-
- Object objectRow[] = { row1, row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public Long getTime() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return 1190592000L;
- } else {
- return 1190505600L;
- }
- }
- };
-
- ExpressionResult result = notEqualsExpression.evaluate(value);
- assertTrue(result.getBoolean());
- } catch (ParseException e) {
- System.out.println("Error while parsing " + e.getMessage());
- }
- }
-
- @Test(expected = FilterUnsupportedException.class) public void testForNotEqualsExpressionWithDefaultCase()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("contact", DataType.BOOLEAN);
- right.setColIndex(0);
- notEqualsExpression = new NotEqualsExpression(right, right);
- RowImpl value = new RowImpl();
- Boolean[] row = { true };
- Object objectRow[] = { row };
- value.setValues(objectRow);
- notEqualsExpression.evaluate(value);
- }
-
- @Test public void testEvaluateWithForNotEqualsExpressionDecimalDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("contact", DataType.DECIMAL);
- right.setColIndex(1);
- ColumnExpression left = new ColumnExpression("contact", DataType.DECIMAL);
- left.setColIndex(0);
- notEqualsExpression = new NotEqualsExpression(left, right);
- RowImpl value = new RowImpl();
- Decimal[] row = new Decimal[] { Decimal.apply(12345.0) };
- Decimal[] row1 = new Decimal[] { Decimal.apply(1235445.0) };
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public BigDecimal getDecimal() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return new BigDecimal(12345.0);
- } else {
- return new BigDecimal(1235445.0);
- }
- }
- };
-
- ExpressionResult result = notEqualsExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForNotEqualsExpressionWithIsNullReturnTrue()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
- right.setColIndex(0);
- notEqualsExpression = new NotEqualsExpression(right, right);
- RowImpl value = new RowImpl();
- Short[] row = { 150 };
- Object objectRow[] = { row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public boolean isNull() {
- return true;
- }
- };
-
- new MockUp<ExpressionResult>() {
- @Mock public Short getShort() {
- return 150;
- }
- };
-
- ExpressionResult result = notEqualsExpression.evaluate(value);
- assertFalse(result.getBoolean());
-
- }
-
- @Test public void testEvaluateForNotEqualsExpressionWithLeftAndRightDifferentDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression left = new ColumnExpression("name", DataType.STRING);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("number", DataType.INT);
- right.setColIndex(1);
- notEqualsExpression = new NotEqualsExpression(left, right);
- RowImpl value = new RowImpl();
- String[] row1 = { "S" };
- Integer[] row = { 14 };
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- Boolean returnMockFlag = true;
-
- @Mock public Integer getInt() {
- if (returnMockFlag) {
- returnMockFlag = false;
- return 84;
- } else {
- return 14;
- }
- }
- };
-
- ExpressionResult result = notEqualsExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testForNotEqualsExpressionWithGetString() throws Exception {
- ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
- ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
- notEqualsExpression = new NotEqualsExpression(left, right);
- String expected_result = "NotEquals(ColumnExpression(left_name),ColumnExpression(right_name))";
- String result = notEqualsExpression.getString();
- assertEquals(expected_result, result);
- }
-
- @Test public void testEvaluateForNotEqualsExpressionWithNullWhileCreatingObject()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
- right.setColIndex(0);
- notEqualsExpression = new NotEqualsExpression(right, right, false);
- RowImpl value = new RowImpl();
- Short[] row = { 15 };
- Object objectRow[] = { row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public boolean isNull() {
- return true;
- }
- };
-
- new MockUp<ExpressionResult>() {
- @Mock public Short getShort() {
- return 15;
- }
- };
-
- ExpressionResult result = notEqualsExpression.evaluate(value);
- assertEquals(DataType.BOOLEAN, result.getDataType());
-
- }
-
- @Test public void testEvaluateForNotEqualsExpressionWithNullISTureWhileCreatingObject()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
- right.setColIndex(0);
- notEqualsExpression = new NotEqualsExpression(right, right, true);
- RowImpl value = new RowImpl();
- Short[] row = { 15 };
- Object objectRow[] = { row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public boolean isNull() {
- return true;
- }
- };
-
- ExpressionResult result = notEqualsExpression.evaluate(value);
- assertEquals(DataType.BOOLEAN, result.getDataType());
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scan/expression/conditional/NotInExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/expression/conditional/NotInExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/scan/expression/conditional/NotInExpressionUnitTest.java
deleted file mode 100644
index 44a144c..0000000
--- a/core/src/test/java/org/apache/carbondata/scan/expression/conditional/NotInExpressionUnitTest.java
+++ /dev/null
@@ -1,275 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.expression.conditional;
-
-import java.math.BigDecimal;
-import java.sql.Timestamp;
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.intf.RowImpl;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.apache.spark.sql.types.Decimal;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-public class NotInExpressionUnitTest {
-
- static NotInExpression notInExpression;
-
- @Test public void testEvaluateForNotInExpressionWithString()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
- right.setColIndex(1);
- notInExpression = new NotInExpression(left, right);
- RowImpl value = new RowImpl();
- String row = "Row is for left";
- String row1 = "I am row 1";
- Object objectRow[] = { row, row1 };
-
- new MockUp<ExpressionResult>() {
-
- @Mock public DataType getDataType() {
- return DataType.STRING;
- }
-
- @Mock public String getString() {
- return "string1";
- }
- };
-
- value.setValues(objectRow);
- ExpressionResult result = notInExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForNotInExpressionWithShortDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
-
- ColumnExpression left = new ColumnExpression("left_id", DataType.SHORT);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("right_id", DataType.SHORT);
- right.setColIndex(1);
- notInExpression = new NotInExpression(left, right);
- RowImpl value = new RowImpl();
- Short row = 15653;
- Short row1 = 15582;
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public Short getShort() {
- return 15582;
- }
- };
-
- ExpressionResult result = notInExpression.evaluate(value);
- assertEquals(result.getDataType(), DataType.BOOLEAN);
-
- }
-
- @Test public void testEvaluateForNotInExpressionWithIntDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
-
- ColumnExpression left = new ColumnExpression("left_id", DataType.INT);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("right_id", DataType.INT);
- right.setColIndex(1);
- notInExpression = new NotInExpression(left, right);
- RowImpl value = new RowImpl();
- Integer row = 150569;
- Integer row1 = 15052;
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public Integer getInt() {
- return 15052;
- }
- };
-
- ExpressionResult result = notInExpression.evaluate(value);
- assertEquals(result.getDataType(), DataType.BOOLEAN);
-
- }
-
- @Test public void testEvaluateForNotInExpressionWithDoubleDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression left = new ColumnExpression("left_contact", DataType.DOUBLE);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("right_contact", DataType.DOUBLE);
- right.setColIndex(1);
- notInExpression = new NotInExpression(left, right);
- RowImpl value = new RowImpl();
- Double row = 44521D;
- Double row1 = 44521.023D;
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public Double getDouble() {
- return 44521.023D;
- }
- };
-
- ExpressionResult result = notInExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForNotInExpressionWithLongDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression left = new ColumnExpression("left_contact", DataType.LONG);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("right_contact", DataType.LONG);
- right.setColIndex(1);
- notInExpression = new NotInExpression(left, right);
- RowImpl value = new RowImpl();
- Long row = 123456256325632L;
- Long row1 = 156212456245556L;
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public Long getLong() {
- return 156212456245556L;
- }
- };
-
- ExpressionResult result = notInExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test public void testEvaluateForNotInExpressionWithTimestampDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- try {
- ColumnExpression left = new ColumnExpression("left_timestamp", DataType.TIMESTAMP);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("right_timestamp", DataType.TIMESTAMP);
- right.setColIndex(1);
- notInExpression = new NotInExpression(right, right);
-
- RowImpl value = new RowImpl();
- DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd");
- Date date = dateFormat.parse("2007/03/03");
-
- long time = date.getTime();
-
- Timestamp row = new Timestamp(time);
- Timestamp row1 = new Timestamp(time);
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public Long getTime() {
- return 1172860200000L;
- }
- };
-
- ExpressionResult result = notInExpression.evaluate(value);
- assertTrue(result.getBoolean());
- } catch (ParseException e) {
- System.out.println("Error while parsing " + e.getMessage());
- }
- }
-
- @Test public void testEvaluateForNotInExpressionWithDecimalDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression left = new ColumnExpression("left_contact", DataType.DECIMAL);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("right_contact", DataType.DECIMAL);
- right.setColIndex(1);
- notInExpression = new NotInExpression(left, right);
- RowImpl value = new RowImpl();
- Decimal row = Decimal.apply(123452154.0);
- Decimal row1 = Decimal.apply(1234521215454.0);
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public BigDecimal getDecimal() {
- return new BigDecimal(1234521215454.0);
- }
- };
-
- ExpressionResult result = notInExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
- @Test(expected = FilterUnsupportedException.class) public void testDefaultCaseForNotInExpression()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression left = new ColumnExpression("contact", DataType.BOOLEAN);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("contact", DataType.BOOLEAN);
- right.setColIndex(1);
- notInExpression = new NotInExpression(left, right);
- RowImpl value = new RowImpl();
- Boolean row = true;
- Boolean row1 = true;
- Object objectRow[] = { row, row1 };
- value.setValues(objectRow);
- notInExpression.evaluate(value);
- }
-
- @Test public void testForNotInExpressionWithGetString() throws Exception {
- ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
- left.setColIndex(0);
- ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
- right.setColIndex(1);
- notInExpression = new NotInExpression(left, right);
- String expected_result = "NOT IN(ColumnExpression(left_name),ColumnExpression(right_name))";
- String result = notInExpression.getString();
- assertEquals(expected_result, result);
- }
- @Test public void testEvaluateForNotInExpressionWithLeftAndRightDifferentDataType()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- ColumnExpression right = new ColumnExpression("name", DataType.STRING);
- right.setColIndex(0);
- ColumnExpression left = new ColumnExpression("number", DataType.INT);
- left.setColIndex(1);
- notInExpression = new NotInExpression(left, right);
- RowImpl value = new RowImpl();
- String row1 = "String1";
- Integer row = 14523213;
- Object objectRow[] = { row1, row };
- value.setValues(objectRow);
-
- new MockUp<ExpressionResult>() {
- @Mock public Integer getInt() {
- return 145232130;
- }
- };
-
- ExpressionResult result = notInExpression.evaluate(value);
- assertTrue(result.getBoolean());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scan/expression/logical/AndExpressionTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/expression/logical/AndExpressionTest.java b/core/src/test/java/org/apache/carbondata/scan/expression/logical/AndExpressionTest.java
deleted file mode 100644
index 2f15f50..0000000
--- a/core/src/test/java/org/apache/carbondata/scan/expression/logical/AndExpressionTest.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression.logical;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.intf.RowImpl;
-import org.apache.carbondata.scan.filter.intf.RowIntf;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.junit.Before;
-import org.junit.Test;
-
-import static junit.framework.TestCase.assertTrue;
-import static org.junit.Assert.assertEquals;
-
-public class AndExpressionTest {
-
- private AndExpression andExpression;
-
- @Before public void setUp() {
- ColumnExpression leftExpression = new ColumnExpression("IMEI", DataType.BOOLEAN);
- ColumnExpression rightExpression = new ColumnExpression("IMEI", DataType.BOOLEAN);
- andExpression = new AndExpression(leftExpression, rightExpression);
- }
-
- @Test(expected = Exception.class) public void testEvaluateForDefault()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- RowImpl rowImpl = new RowImpl();
- rowImpl.setValues(new Boolean[] { true });
- final ExpressionResult expressionResult = new ExpressionResult(DataType.STRING, "test");
- new MockUp<ColumnExpression>() {
- @Mock public ExpressionResult evaluate(RowIntf value)
- throws FilterUnsupportedException, FilterIllegalMemberException {
- return expressionResult;
- }
- };
- andExpression.evaluate(rowImpl);
- }
-
- @Test public void testEvaluate() throws FilterUnsupportedException, FilterIllegalMemberException {
- RowImpl rowImpl = new RowImpl();
- rowImpl.setValues(new Boolean[] { false });
- final ExpressionResult expressionResult = new ExpressionResult(DataType.BOOLEAN, "test");
- new MockUp<ColumnExpression>() {
- @Mock public ExpressionResult evaluate(RowIntf value)
- throws FilterUnsupportedException, FilterIllegalMemberException {
- return expressionResult;
- }
- };
- ExpressionResult actualValue = andExpression.evaluate(rowImpl);
- assertTrue(actualValue instanceof ExpressionResult);
- }
-
- @Test public void testGetString() {
- String actualValue = andExpression.getString();
- String expectedValue = "And(ColumnExpression(IMEI),ColumnExpression(IMEI))";
- assertEquals(expectedValue, actualValue);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scan/expression/logical/FalseExpressionTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/expression/logical/FalseExpressionTest.java b/core/src/test/java/org/apache/carbondata/scan/expression/logical/FalseExpressionTest.java
deleted file mode 100644
index 6ec0b59..0000000
--- a/core/src/test/java/org/apache/carbondata/scan/expression/logical/FalseExpressionTest.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression.logical;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.intf.RowImpl;
-
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-public class FalseExpressionTest {
- private FalseExpression falseExpression;
-
- @Before public void setUp() {
- ColumnExpression columnExpression = new ColumnExpression("IMEI", DataType.BOOLEAN);
- falseExpression = new FalseExpression(columnExpression);
- }
-
- @Test public void testEvaluate() throws FilterUnsupportedException, FilterIllegalMemberException {
- RowImpl rowImpl = new RowImpl();
- rowImpl.setValues(new Boolean[] { true });
- ExpressionResult actualValue = falseExpression.evaluate(rowImpl);
- assertEquals(new ExpressionResult(DataType.BOOLEAN, false), actualValue);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scan/expression/logical/OrExpressionTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/expression/logical/OrExpressionTest.java b/core/src/test/java/org/apache/carbondata/scan/expression/logical/OrExpressionTest.java
deleted file mode 100644
index 44b24e5..0000000
--- a/core/src/test/java/org/apache/carbondata/scan/expression/logical/OrExpressionTest.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.expression.logical;
-
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.ExpressionResult;
-import org.apache.carbondata.scan.expression.exception.FilterIllegalMemberException;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.intf.RowImpl;
-import org.apache.carbondata.scan.filter.intf.RowIntf;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.junit.Before;
-import org.junit.Test;
-
-import static junit.framework.TestCase.assertTrue;
-import static org.junit.Assert.assertEquals;
-
-public class OrExpressionTest {
- private OrExpression orExpression;
-
- @Before public void setUp() {
- ColumnExpression leftExpression = new ColumnExpression("IMEI", DataType.BOOLEAN);
- ColumnExpression rightExpression = new ColumnExpression("IMEI", DataType.BOOLEAN);
- orExpression = new OrExpression(leftExpression, rightExpression);
- }
-
- @Test public void testGetString() {
- String actualValue = orExpression.getString();
- String expectedValue = "Or(ColumnExpression(IMEI),ColumnExpression(IMEI))";
- assertEquals(expectedValue, actualValue);
- }
-
- @Test public void testEvaluate() throws FilterIllegalMemberException, FilterUnsupportedException {
- RowImpl rowImpl = new RowImpl();
- rowImpl.setValues(new Boolean[] { false });
- final ExpressionResult expressionResult = new ExpressionResult(DataType.BOOLEAN, "test");
- new MockUp<ColumnExpression>() {
- @Mock public ExpressionResult evaluate(RowIntf value) {
- return expressionResult;
- }
- };
-
- assertTrue(orExpression.evaluate(rowImpl) instanceof ExpressionResult);
- }
-
- @Test(expected = Exception.class) public void testEvaluateForDefault()
- throws FilterUnsupportedException, FilterIllegalMemberException {
- RowImpl rowImpl = new RowImpl();
- rowImpl.setValues(new Boolean[] { true });
- final ExpressionResult expressionResult = new ExpressionResult(DataType.STRING, "test");
- new MockUp<ColumnExpression>() {
- @Mock public ExpressionResult evaluate(RowIntf value)
- throws FilterUnsupportedException, FilterIllegalMemberException {
- return expressionResult;
- }
- };
- orExpression.evaluate(rowImpl);
- }
-}
[32/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinByte.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinByte.java
new file mode 100644
index 0000000..74d67d1
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinByte.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.compression.nondecimal;
+
+import java.math.BigDecimal;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastorage.compression.Compressor;
+import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNonDecimalMaxMinByte extends ValueCompressionHolder<byte[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNonDecimalMaxMinByte.class.getName());
+
+ /**
+ * compressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+
+ /**
+ * value.
+ */
+ private byte[] value;
+
+ private MeasureDataChunkStore<byte[]> measureChunkStore;
+
+ private BigDecimal maxValue;
+
+ private double divisionFactor;
+
+ @Override
+ public void uncompress(DataType dataType, byte[] compressedData,
+ int offset, int length, int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, compressedData, offset, length);
+ setUncompressedValues(value, decimalPlaces, maxValueObject );
+ }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_BYTE, value);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ this.value = value;
+ }
+
+ @Override public void setValue(byte[] value) {
+ this.value = value;
+ }
+
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException(
+ "Long value is not defined for CompressionNonDecimalMaxMinByte");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ byte byteValue = measureChunkStore.getByte(index);
+ BigDecimal diff = BigDecimal.valueOf(byteValue / this.divisionFactor);
+ return maxValue.subtract(diff).doubleValue();
+
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal value is not defined for CompressionNonDecimalMaxMinByte");
+ }
+
+ private void setUncompressedValues(byte[] data, int decimalPlaces, Object maxValueObject) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_BYTE, data.length);
+ this.measureChunkStore.putData(data);
+ this.maxValue = BigDecimal.valueOf((double) maxValueObject);
+ this.divisionFactor = Math.pow(10, decimalPlaces);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+
+ @Override public byte[] getValue() { return this.value; }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java
new file mode 100644
index 0000000..e298c37
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.compression.nondecimal;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastorage.compression.Compressor;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNonDecimalMaxMinDefault extends ValueCompressionHolder<double[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNonDecimalMaxMinDefault.class.getName());
+
+ /**
+ * doubleCompressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+
+ /**
+ * value.
+ */
+ private double[] value;
+
+ private MeasureDataChunkStore<double[]> measureChunkStore;
+
+ private BigDecimal maxValue;
+
+ private double divisionFactor;
+
+ @Override public void setValue(double[] value) {
+ this.value = value;
+ }
+
+ @Override public double[] getValue() { return this.value; }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_DOUBLE, value);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ ByteBuffer buffer = ByteBuffer.wrap(value);
+ this.value = ValueCompressionUtil.convertToDoubleArray(buffer, value.length);
+ }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] compressedData,
+ int offset, int length, int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor,dataType,compressedData, offset, length);
+ setUncompressedValues(value, decimalPlaces, maxValueObject);
+ }
+
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException(
+ "Long value is not defined for CompressionNonDecimalMaxMinDefault");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ double doubleValue = measureChunkStore.getDouble(index);
+ BigDecimal diff = BigDecimal.valueOf(doubleValue / this.divisionFactor);
+ return maxValue.subtract(diff).doubleValue();
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal value is not defined for CompressionNonDecimalMaxMinDefault");
+ }
+
+ private void setUncompressedValues(double[] data, int decimalPlaces, Object maxValueObject) {
+ this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
+ .getMeasureDataChunkStore(DataType.DATA_DOUBLE, data.length);
+ this.measureChunkStore.putData(data);
+ this.maxValue = BigDecimal.valueOf((double) maxValueObject);
+ this.divisionFactor = Math.pow(10, decimalPlaces);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinInt.java
new file mode 100644
index 0000000..fc0f815
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinInt.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.compression.nondecimal;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastorage.compression.Compressor;
+import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNonDecimalMaxMinInt extends ValueCompressionHolder<int[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNonDecimalMaxMinInt.class.getName());
+ /**
+ * intCompressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+ /**
+ * value.
+ */
+ private int[] value;
+
+ private MeasureDataChunkStore<int[]> measureChunkStore;
+
+ private double divisionFactor;
+
+ private BigDecimal maxValue;
+
+ @Override public void setValue(int[] value) {
+ this.value = value;
+ }
+
+ @Override public int[] getValue() {return this.value; }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_INT, value);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ ByteBuffer buffer = ByteBuffer.wrap(value);
+ this.value = ValueCompressionUtil.convertToIntArray(buffer, value.length);
+ }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] compressedData,
+ int offset, int length, int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, compressedData, offset, length);
+ setUncompressedValues(value, decimalPlaces, maxValueObject);
+ }
+
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException(
+ "Long value is not defined for CompressionNonDecimalMaxMinInt");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ int intValue = measureChunkStore.getInt(index);
+ BigDecimal diff = BigDecimal.valueOf(intValue / this.divisionFactor);
+ return maxValue.subtract(diff).doubleValue();
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal value is not defined for CompressionNonDecimalMaxMinInt");
+ }
+
+ private void setUncompressedValues(int[] data, int decimalPlaces, Object maxValueObject) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, data.length);
+ this.measureChunkStore.putData(data);
+ this.maxValue = BigDecimal.valueOf((double) maxValueObject);
+ this.divisionFactor = Math.pow(10, decimalPlaces);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinLong.java
new file mode 100644
index 0000000..8524c20
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinLong.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.compression.nondecimal;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastorage.compression.Compressor;
+import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNonDecimalMaxMinLong extends ValueCompressionHolder<long[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNonDecimalMaxMinLong.class.getName());
+
+ /**
+ * longCompressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+ /**
+ * value.
+ */
+ private long[] value;
+
+ private MeasureDataChunkStore<long[]> measureChunkStore;
+
+ private BigDecimal maxValue;
+
+ private double divisionFactor;
+
+ @Override public void setValue(long[] value) {
+ this.value = value;
+
+ }
+
+ @Override public long[] getValue() { return this.value; }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] compressedData,
+ int offset, int length, int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, compressedData, offset, length);
+ setUncompressedValues(value, decimalPlaces, maxValueObject);
+ }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_LONG, value);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ ByteBuffer buff = ByteBuffer.wrap(value);
+ this.value = ValueCompressionUtil.convertToLongArray(buff, value.length);
+ }
+
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException(
+ "Long value is not defined for CompressionNonDecimalMaxMinLong");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ long longValue = measureChunkStore.getLong(index);
+ BigDecimal diff = BigDecimal.valueOf(longValue / this.divisionFactor);
+ return maxValue.subtract(diff).doubleValue();
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException("Big decimal value is not supported");
+ }
+
+ private void setUncompressedValues(long[] data, int decimalPlaces, Object maxValueObject) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_LONG, data.length);
+ this.measureChunkStore.putData(data);
+ this.maxValue = BigDecimal.valueOf((double) maxValueObject);
+ this.divisionFactor = Math.pow(10, decimalPlaces);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinShort.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinShort.java
new file mode 100644
index 0000000..dac557c
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinShort.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.compression.nondecimal;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastorage.compression.Compressor;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNonDecimalMaxMinShort extends ValueCompressionHolder<short[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNonDecimalMaxMinShort.class.getName());
+ /**
+ * compressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+ /**
+ * value.
+ */
+ private short[] value;
+
+ private MeasureDataChunkStore<short[]> measureChunkStore;
+
+ private BigDecimal maxValue;
+
+ private double divisionFactor;
+
+ @Override public void setValue(short[] value) {
+ this.value = value;
+ }
+
+ @Override public short[] getValue() { return this.value; }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] compressedData,
+ int offset, int length, int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, compressedData, offset, length);
+ setUncompressedValues(value, decimalPlaces, maxValueObject);
+ }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_SHORT, value);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ ByteBuffer buffer = ByteBuffer.wrap(value);
+ this.value = ValueCompressionUtil.convertToShortArray(buffer, value.length);
+ }
+
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException(
+ "Long value is not defined for CompressionNonDecimalMaxMinShort");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ short shortValue = measureChunkStore.getShort(index);
+ BigDecimal diff = BigDecimal.valueOf(shortValue / this.divisionFactor);
+ return maxValue.subtract(diff).doubleValue();
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException("Get big decimal value is not supported");
+ }
+
+ private void setUncompressedValues(short[] data, int decimalPlaces, Object maxValueObject) {
+ this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
+ .getMeasureDataChunkStore(DataType.DATA_SHORT, data.length);
+ this.measureChunkStore.putData(data);
+ this.maxValue = BigDecimal.valueOf((double) maxValueObject);
+ this.divisionFactor = Math.pow(10, decimalPlaces);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalShort.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalShort.java
new file mode 100644
index 0000000..334d7e8
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalShort.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.compression.nondecimal;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastorage.compression.Compressor;
+import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNonDecimalShort extends ValueCompressionHolder<short[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNonDecimalShort.class.getName());
+ /**
+ * shortCompressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+ /**
+ * value.
+ */
+ private short[] value;
+
+ private MeasureDataChunkStore<short[]> measureChunkStore;
+
+ private double divisionFactory;
+
+ @Override public void setValue(short[] value) {
+ this.value = value;
+ }
+
+ @Override public short[] getValue() { return this.value; }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_SHORT, value);
+ }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] compressedData,
+ int offset, int length, int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor,dataType,compressedData, offset, length);
+ setUncompressedValues(value, decimalPlaces);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ ByteBuffer buffer = ByteBuffer.wrap(value);
+ this.value = ValueCompressionUtil.convertToShortArray(buffer, value.length);
+ }
+
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException(
+ "Long value is not defined for CompressionNonDecimalShort");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ return (measureChunkStore.getShort(index) / this.divisionFactory);
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal value is not defined for CompressionNonDecimalShort");
+ }
+
+ private void setUncompressedValues(short[] data, int decimalPlaces) {
+ this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
+ .getMeasureDataChunkStore(DataType.DATA_SHORT, data.length);
+ this.measureChunkStore.putData(data);
+ this.divisionFactory = Math.pow(10, decimalPlaces);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneByte.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneByte.java
new file mode 100644
index 0000000..308ec8b
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneByte.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.compression.none;
+
+import java.math.BigDecimal;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastorage.compression.Compressor;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNoneByte extends ValueCompressionHolder<byte[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNoneByte.class.getName());
+
+ /**
+ * byteCompressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+
+ /**
+ * value.
+ */
+ private byte[] value;
+
+ /**
+ * actual data type
+ */
+ private DataType actualDataType;
+
+ private MeasureDataChunkStore<byte[]> measureChunkStore;
+
+ public CompressionNoneByte(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
+ @Override public void setValue(byte[] value) {
+ this.value = value;
+ }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] data, int offset, int length,
+ int mantissa, Object maxValueObject) {
+ super.unCompress(compressor, dataType, data, offset, length);
+ setUncompressedValues(value);
+ }
+
+ @Override public byte[] getValue() { return this.value; }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_BYTE, value);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ this.value = value;
+ }
+
+ @Override public long getLongValue(int index) {
+ return measureChunkStore.getByte(index);
+ }
+
+ @Override public double getDoubleValue(int index) {
+ return measureChunkStore.getByte(index);
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal is not defined for CompressionNoneByte");
+ }
+
+ private void setUncompressedValues(byte[] data) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_BYTE, data.length);
+ this.measureChunkStore.putData(data);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneDefault.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneDefault.java
new file mode 100644
index 0000000..65fa3e5
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneDefault.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.compression.none;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastorage.compression.Compressor;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNoneDefault extends ValueCompressionHolder<double[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNoneDefault.class.getName());
+ /**
+ * doubleCompressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+ /**
+ * value.
+ */
+ private double[] value;
+
+ private DataType actualDataType;
+
+ private MeasureDataChunkStore<double[]> measureChunkStore;
+
+ public CompressionNoneDefault(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
+ @Override public void setValue(double[] value) {this.value = value; }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] data, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, data, offset, length);
+ setUncompressedValues(value);
+ }
+
+ @Override public double[] getValue() { return this.value; }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_DOUBLE, value);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ ByteBuffer buffer = ByteBuffer.wrap(value);
+ this.value = ValueCompressionUtil.convertToDoubleArray(buffer, value.length);
+ }
+
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException(
+ "Long value is not defined for CompressionNonDefault");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ return measureChunkStore.getDouble(index);
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal is not defined for CompressionNoneDefault");
+ }
+
+ private void setUncompressedValues(double[] data) {
+ this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
+ .getMeasureDataChunkStore(DataType.DATA_DOUBLE, data.length);
+ this.measureChunkStore.putData(data);
+
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneInt.java
new file mode 100644
index 0000000..29a60cf
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneInt.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.compression.none;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastorage.compression.Compressor;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNoneInt extends ValueCompressionHolder<int[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNoneInt.class.getName());
+ /**
+ * intCompressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+ /**
+ * value.
+ */
+ private int[] value;
+
+ private DataType actualDataType;
+
+ private MeasureDataChunkStore<int[]> measureChunkStore;
+
+ public CompressionNoneInt(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
+ @Override public void setValue(int[] value) {
+ this.value = value;
+ }
+
+ @Override public int[] getValue() { return this.value; }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_INT, value);
+ }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] data, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, data, offset, length);
+ setUncompressedValues(value);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ ByteBuffer buffer = ByteBuffer.wrap(value);
+ this.value = ValueCompressionUtil.convertToIntArray(buffer, value.length);
+ }
+
+ @Override public long getLongValue(int index) {
+ return measureChunkStore.getInt(index);
+ }
+
+ @Override public double getDoubleValue(int index) {
+ return measureChunkStore.getInt(index);
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal is not defined for CompressionNoneLong");
+ }
+
+ private void setUncompressedValues(int[] data) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, data.length);
+ this.measureChunkStore.putData(data);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneLong.java
new file mode 100644
index 0000000..834467d
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneLong.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.compression.none;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastorage.compression.Compressor;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNoneLong extends ValueCompressionHolder<long[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNoneLong.class.getName());
+ /**
+ * longCompressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+ /**
+ * value.
+ */
+ protected long[] value;
+
+ private DataType actualDataType;
+
+ private MeasureDataChunkStore<long[]> measureChunkStore;
+
+ public CompressionNoneLong(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
+ @Override public void setValue(long[] value) { this.value = value; }
+
+ @Override public long[] getValue() { return this.value; }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_LONG, value);
+ }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] data, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, data, offset, length);
+ setUncompressedValues(value);
+ }
+
+ @Override public void setValueInBytes(byte[] byteValue) {
+ ByteBuffer buffer = ByteBuffer.wrap(byteValue);
+ this.value = ValueCompressionUtil.convertToLongArray(buffer, byteValue.length);
+ }
+
+ @Override public long getLongValue(int index) {
+ return measureChunkStore.getLong(index);
+ }
+
+ @Override public double getDoubleValue(int index) {
+ return measureChunkStore.getLong(index);
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException("Get big decimal is not supported");
+ }
+
+ private void setUncompressedValues(long[] data) {
+ this.measureChunkStore =
+ MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_LONG, data.length);
+ this.measureChunkStore.putData(data);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneShort.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneShort.java
new file mode 100644
index 0000000..f0b1c99
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneShort.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.compression.none;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastorage.compression.Compressor;
+import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+public class CompressionNoneShort extends ValueCompressionHolder<short[]> {
+ /**
+ * Attribute for Carbon LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionNoneShort.class.getName());
+
+ /**
+ * shortCompressor.
+ */
+ private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
+
+ /**
+ * value.
+ */
+ private short[] shortValue;
+
+ private MeasureDataChunkStore<short[]> measureChunkStore;
+
+ private DataType actualDataType;
+
+ public CompressionNoneShort(DataType actualDataType) {
+ this.actualDataType = actualDataType;
+ }
+
+ @Override public void setValue(short[] shortValue) {
+ this.shortValue = shortValue;
+ }
+
+ @Override public short[] getValue() { return this.shortValue; }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] data, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
+ super.unCompress(compressor, dataType, data, offset, length);
+ setUncompressedValues(shortValue);
+ }
+
+ @Override public void compress() {
+ compressedValue = super.compress(compressor, DataType.DATA_SHORT, shortValue);
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ ByteBuffer buffer = ByteBuffer.wrap(value);
+ shortValue = ValueCompressionUtil.convertToShortArray(buffer, value.length);
+ }
+
+ @Override public long getLongValue(int index) {
+ return measureChunkStore.getShort(index);
+ }
+
+ @Override public double getDoubleValue(int index) {
+ return measureChunkStore.getShort(index);
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ throw new UnsupportedOperationException(
+ "Big decimal is not defined for CompressionNonShort");
+ }
+
+ private void setUncompressedValues(short[] data) {
+ this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
+ .getMeasureDataChunkStore(DataType.DATA_SHORT, data.length);
+ this.measureChunkStore.putData(data);
+ }
+
+ @Override public void freeMemory() {
+ this.measureChunkStore.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/compression/type/CompressionBigDecimal.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/type/CompressionBigDecimal.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/type/CompressionBigDecimal.java
new file mode 100644
index 0000000..8233e90
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/type/CompressionBigDecimal.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.datastorage.compression.type;
+
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.util.BigDecimalCompressionFinder;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+/**
+ * Big decimal compression
+ */
+public class CompressionBigDecimal<T> extends ValueCompressionHolder<T> {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CompressionBigDecimal.class.getName());
+
+ private BigDecimalCompressionFinder compressionFinder;
+
+ /**
+ * leftPart before decimal
+ */
+ private ValueCompressionHolder leftPart;
+
+ /**
+ * rightPart after decimal
+ */
+ private ValueCompressionHolder rightPart;
+
+ private double divisionFactor;
+
+ private boolean isDecimalPlacesNotZero;
+
+ public CompressionBigDecimal(BigDecimalCompressionFinder compressionFinder,
+ ValueCompressionHolder leftPart, ValueCompressionHolder rightPart) {
+ this.compressionFinder = compressionFinder;
+ this.leftPart = leftPart;
+ this.rightPart = rightPart;
+ }
+
+ @Override public void setValue(T value) {
+ Object[] values = (Object[]) value;
+ leftPart.setValue(values[0]);
+ rightPart.setValue(values[1]);
+ }
+
+ @Override
+ public void uncompress(DataType dataType, byte[] data, int offset, int length,
+ int decimalPlaces, Object maxValueObject) {
+ if (decimalPlaces > 0) {
+ this.isDecimalPlacesNotZero = true;
+ }
+ this.divisionFactor = Math.pow(10, decimalPlaces);
+
+ ByteBuffer buffer = ByteBuffer.wrap(data, offset, length);
+ int leftPathLength = buffer.getInt();
+ int rightPartLength = length - leftPathLength - CarbonCommonConstants.INT_SIZE_IN_BYTE;
+ Long[] maxValue = (Long[]) maxValueObject;
+ leftPart.uncompress(compressionFinder.getLeftConvertedDataType(), data,
+ offset + CarbonCommonConstants.INT_SIZE_IN_BYTE, leftPathLength, decimalPlaces,
+ maxValue[0]);
+ rightPart.uncompress(compressionFinder.getRightConvertedDataType(), data,
+ offset + CarbonCommonConstants.INT_SIZE_IN_BYTE + leftPathLength, rightPartLength,
+ decimalPlaces, maxValue[1]);
+ }
+
+ @Override public long getLongValue(int index) {
+ throw new UnsupportedOperationException(
+ "Long is not defined for CompressionBigDecimal");
+ }
+
+ @Override public double getDoubleValue(int index) {
+ throw new UnsupportedOperationException(
+ "Double is not defined for CompressionBigDecimal");
+ }
+
+ @Override public BigDecimal getBigDecimalValue(int index) {
+ long leftValue = leftPart.getLongValue(index);
+ long rightValue = 0;
+ if (isDecimalPlacesNotZero) {
+ rightValue = rightPart.getLongValue(index);
+ }
+ String decimalPart = Double.toString(rightValue / this.divisionFactor);
+ String bigdStr = Long.toString(leftValue) + CarbonCommonConstants.POINT + decimalPart
+ .substring(decimalPart.indexOf(".") + 1, decimalPart.length());
+ return new BigDecimal(bigdStr);
+ }
+
+ @Override public T getValue() {
+ Object[] values = new Object[2];
+ values[0] = leftPart;
+ values[1] = rightPart;
+ return (T) values;
+ }
+
+ @Override public void setValueInBytes(byte[] value) {
+ LOGGER.error("setValueInBytes() is not defined for CompressionBigDecimal");
+ }
+
+ @Override public void compress() {
+ leftPart.compress();
+ rightPart.compress();
+ }
+
+ @Override
+ public byte[] getCompressedData() {
+ byte[] leftdata = leftPart.getCompressedData();
+ byte[] rightdata = rightPart.getCompressedData();
+ ByteBuffer byteBuffer = ByteBuffer
+ .allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE + leftdata.length
+ + rightdata.length);
+ byteBuffer.putInt(leftdata.length);
+ byteBuffer.put(leftdata);
+ byteBuffer.put(rightdata);
+ byteBuffer.flip();
+ return byteBuffer.array();
+ }
+
+ @Override public void freeMemory() {
+ leftPart.freeMemory();
+ rightPart.freeMemory();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/dataholder/CarbonReadDataHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/dataholder/CarbonReadDataHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/dataholder/CarbonReadDataHolder.java
new file mode 100644
index 0000000..29ff254
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/dataholder/CarbonReadDataHolder.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.dataholder;
+
+import java.math.BigDecimal;
+
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+
+// This class is used with Uncompressor to hold the decompressed column chunk in memory
+public class CarbonReadDataHolder {
+
+ private ValueCompressionHolder unCompressValue;
+
+ public CarbonReadDataHolder(ValueCompressionHolder unCompressValue) {
+ this.unCompressValue = unCompressValue;
+ }
+
+ public long getReadableLongValueByIndex(int index) {
+ return this.unCompressValue.getLongValue(index);
+ }
+
+ public BigDecimal getReadableBigDecimalValueByIndex(int index) {
+ return this.unCompressValue.getBigDecimalValue(index);
+ }
+
+ public double getReadableDoubleValueByIndex(int index) {
+ return this.unCompressValue.getDoubleValue(index);
+ }
+
+ public void freeMemory() {
+ unCompressValue.freeMemory();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/dataholder/CarbonWriteDataHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/dataholder/CarbonWriteDataHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/dataholder/CarbonWriteDataHolder.java
new file mode 100644
index 0000000..e6d9123
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/dataholder/CarbonWriteDataHolder.java
@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.dataholder;
+
+public class CarbonWriteDataHolder {
+ /**
+ * doubleValues
+ */
+ private double[] doubleValues;
+
+ /**
+ * longValues
+ */
+ private long[] longValues;
+
+ /**
+ * bigDecimal left part
+ */
+ private long[] bigDecimalLeftValues;
+
+ /**
+ * bigDecimal right part
+ */
+ private long[] bigDecimalRightValues;
+ /**
+ * byteValues
+ */
+ private byte[][] byteValues;
+
+ /**
+ * byteValues for no dictionary and non kettle flow.
+ */
+ private byte[][][] byteValuesForNonDictionary;
+
+ /**
+ * byteValues
+ */
+ private byte[][][] columnByteValues;
+
+ /**
+ * size
+ */
+ private int size;
+
+ /**
+ * totalSize
+ */
+ private int totalSize;
+
+ /**
+ * Method to initialise double array
+ *
+ * @param size
+ */
+ public void initialiseDoubleValues(int size) {
+ if (size < 1) {
+ throw new IllegalArgumentException("Invalid array size");
+ }
+ doubleValues = new double[size];
+ }
+
+ public void reset() {
+ size = 0;
+ totalSize = 0;
+ }
+
+ /**
+ * Method to initialise double array
+ * TODO Remove after kettle flow got removed.
+ *
+ * @param size
+ */
+ public void initialiseByteArrayValues(int size) {
+ if (size < 1) {
+ throw new IllegalArgumentException("Invalid array size");
+ }
+
+ byteValues = new byte[size][];
+ columnByteValues = new byte[size][][];
+ }
+
+ /**
+ * Method to initialise byte array
+ *
+ * @param size
+ */
+ public void initialiseByteArrayValuesWithOutKettle(int size) {
+ if (size < 1) {
+ throw new IllegalArgumentException("Invalid array size");
+ }
+
+ byteValues = new byte[size][];
+ }
+
+ public void initialiseByteArrayValuesForNonDictionary(int size) {
+ if (size < 1) {
+ throw new IllegalArgumentException("Invalid array size");
+ }
+
+ byteValuesForNonDictionary = new byte[size][][];
+ }
+
+ /**
+ * Method to initialise long array
+ *
+ * @param size
+ */
+ public void initialiseLongValues(int size) {
+ if (size < 1) {
+ throw new IllegalArgumentException("Invalid array size");
+ }
+ longValues = new long[size];
+ }
+
+ public void initialiseBigDecimalValues(int size) {
+ if (size < 1) {
+ throw new IllegalArgumentException("Invalid array size");
+ }
+ bigDecimalLeftValues = new long[size];
+ bigDecimalRightValues = new long[size];
+ }
+
+ /**
+ * set double value by index
+ *
+ * @param index
+ * @param value
+ */
+ public void setWritableDoubleValueByIndex(int index, Object value) {
+ doubleValues[index] = (Double) value;
+ size++;
+ }
+
+ /**
+ * set double value by index
+ *
+ * @param index
+ * @param value
+ */
+ public void setWritableLongValueByIndex(int index, Object value) {
+ longValues[index] = (Long) value;
+ size++;
+ }
+
+ /**
+ * set bigdecimal value by index
+ *
+ * @param index
+ * @param value
+ */
+ public void setWritableBigDecimalValueByIndex(int index, long[] value) {
+ bigDecimalLeftValues[index] = value[0];
+ bigDecimalRightValues[index] = value[1];
+ size++;
+ }
+ /**
+ * set byte array value by index
+ *
+ * @param index
+ * @param value
+ */
+ public void setWritableByteArrayValueByIndex(int index, byte[] value) {
+ byteValues[index] = value;
+ size++;
+ if (null != value) totalSize += value.length;
+ }
+
+ public void setWritableNonDictByteArrayValueByIndex(int index, byte[][] value) {
+ byteValuesForNonDictionary[index] = value;
+ size++;
+ if (null != value) totalSize += value.length;
+ }
+
+ /**
+ * set byte array value by index
+ */
+ public void setWritableByteArrayValueByIndex(int index, int mdKeyIndex, Object[] columnData) {
+ int l = 0;
+ columnByteValues[index] = new byte[columnData.length - (mdKeyIndex + 1)][];
+ for (int i = mdKeyIndex + 1; i < columnData.length; i++) {
+ columnByteValues[index][l++] = (byte[]) columnData[i];
+ }
+ }
+
+ /**
+ * Get Writable Double Values
+ */
+ public double[] getWritableDoubleValues() {
+ if (size < doubleValues.length) {
+ double[] temp = new double[size];
+ System.arraycopy(doubleValues, 0, temp, 0, size);
+ doubleValues = temp;
+ }
+ return doubleValues;
+ }
+
+ /**
+ * Get writable byte array values
+ */
+ public byte[] getWritableByteArrayValues() {
+ byte[] temp = new byte[totalSize];
+ int startIndexToCopy = 0;
+ for (int i = 0; i < size; i++) {
+ System.arraycopy(byteValues[i], 0, temp, startIndexToCopy, byteValues[i].length);
+ startIndexToCopy += byteValues[i].length;
+ }
+ return temp;
+ }
+
+ public byte[][] getByteArrayValues() {
+ if (size < byteValues.length) {
+ byte[][] temp = new byte[size][];
+ System.arraycopy(byteValues, 0, temp, 0, size);
+ byteValues = temp;
+ }
+ return byteValues;
+ }
+
+ public byte[][][] getNonDictByteArrayValues() {
+ if (size < byteValuesForNonDictionary.length) {
+ byte[][][] temp = new byte[size][][];
+ System.arraycopy(byteValuesForNonDictionary, 0, temp, 0, size);
+ byteValuesForNonDictionary = temp;
+ }
+ return byteValuesForNonDictionary;
+ }
+
+ /**
+ * Get Writable Double Values
+ *
+ * @return
+ */
+ public long[] getWritableLongValues() {
+ if (size < longValues.length) {
+ long[] temp = new long[size];
+ System.arraycopy(longValues, 0, temp, 0, size);
+ longValues = temp;
+ }
+ return longValues;
+ }
+
+ /**
+ * Get Writable bigdecimal Values
+ *
+ * @return
+ */
+ public long[][] getWritableBigDecimalValues() {
+ long[][] bigDecimalValues = new long[2][];
+ if (size < bigDecimalLeftValues.length) {
+ long[] temp = new long[size];
+ System.arraycopy(bigDecimalLeftValues, 0, temp, 0, size);
+ bigDecimalLeftValues = temp;
+ }
+ if (size < bigDecimalRightValues.length) {
+ long[] temp = new long[size];
+ System.arraycopy(bigDecimalRightValues, 0, temp, 0, size);
+ bigDecimalRightValues = temp;
+ }
+ bigDecimalValues[0]= bigDecimalLeftValues;
+ bigDecimalValues[1] = bigDecimalRightValues;
+ return bigDecimalValues;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/AbstractDFSCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/AbstractDFSCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/AbstractDFSCarbonFile.java
new file mode 100644
index 0000000..8963702
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/AbstractDFSCarbonFile.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.filesystem;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+public abstract class AbstractDFSCarbonFile implements CarbonFile {
+ /**
+ * LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(AbstractDFSCarbonFile.class.getName());
+ protected FileStatus fileStatus;
+ protected FileSystem fs;
+
+ public AbstractDFSCarbonFile(String filePath) {
+ filePath = filePath.replace("\\", "/");
+ Path path = new Path(filePath);
+ try {
+ fs = path.getFileSystem(FileFactory.getConfiguration());
+ fileStatus = fs.getFileStatus(path);
+ } catch (IOException e) {
+ LOGGER.error("Exception occured:" + e.getMessage());
+ }
+ }
+
+ public AbstractDFSCarbonFile(Path path) {
+ try {
+ fs = path.getFileSystem(FileFactory.getConfiguration());
+ fileStatus = fs.getFileStatus(path);
+ } catch (IOException e) {
+ LOGGER.error("Exception occured:" + e.getMessage());
+ }
+ }
+
+ public AbstractDFSCarbonFile(FileStatus fileStatus) {
+ this.fileStatus = fileStatus;
+ }
+
+ @Override public boolean createNewFile() {
+ Path path = fileStatus.getPath();
+ try {
+ return fs.createNewFile(path);
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ @Override public String getAbsolutePath() {
+ return fileStatus.getPath().toString();
+ }
+
+ @Override public String getName() {
+ return fileStatus.getPath().getName();
+ }
+
+ @Override public boolean isDirectory() {
+ return fileStatus.isDirectory();
+ }
+
+ @Override public boolean exists() {
+ try {
+ if (null != fileStatus) {
+ fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
+ return fs.exists(fileStatus.getPath());
+ }
+ } catch (IOException e) {
+ LOGGER.error("Exception occured:" + e.getMessage());
+ }
+ return false;
+ }
+
+ @Override public String getCanonicalPath() {
+ return getAbsolutePath();
+ }
+
+ @Override public String getPath() {
+ return getAbsolutePath();
+ }
+
+ @Override public long getSize() {
+ return fileStatus.getLen();
+ }
+
+ public boolean renameTo(String changetoName) {
+ FileSystem fs;
+ try {
+ fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
+ return fs.rename(fileStatus.getPath(), new Path(changetoName));
+ } catch (IOException e) {
+ LOGGER.error("Exception occured:" + e.getMessage());
+ return false;
+ }
+ }
+
+ public boolean delete() {
+ FileSystem fs;
+ try {
+ fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
+ return fs.delete(fileStatus.getPath(), true);
+ } catch (IOException e) {
+ LOGGER.error("Exception occured:" + e.getMessage());
+ return false;
+ }
+ }
+
+ @Override public long getLastModifiedTime() {
+ return fileStatus.getModificationTime();
+ }
+
+ @Override public boolean setLastModifiedTime(long timestamp) {
+ try {
+ fs.setTimes(fileStatus.getPath(), timestamp, timestamp);
+ } catch (IOException e) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * This method will delete the data in file data from a given offset
+ */
+ @Override public boolean truncate(String fileName, long validDataEndOffset) {
+ DataOutputStream dataOutputStream = null;
+ DataInputStream dataInputStream = null;
+ boolean fileTruncatedSuccessfully = false;
+ // if bytes to read less than 1024 then buffer size should be equal to the given offset
+ int bufferSize = validDataEndOffset > CarbonCommonConstants.BYTE_TO_KB_CONVERSION_FACTOR ?
+ CarbonCommonConstants.BYTE_TO_KB_CONVERSION_FACTOR :
+ (int) validDataEndOffset;
+ // temporary file name
+ String tempWriteFilePath = fileName + CarbonCommonConstants.TEMPWRITEFILEEXTENSION;
+ FileFactory.FileType fileType = FileFactory.getFileType(fileName);
+ try {
+ CarbonFile tempFile = null;
+ // delete temporary file if it already exists at a given path
+ if (FileFactory.isFileExist(tempWriteFilePath, fileType)) {
+ tempFile = FileFactory.getCarbonFile(tempWriteFilePath, fileType);
+ tempFile.delete();
+ }
+ // create new temporary file
+ FileFactory.createNewFile(tempWriteFilePath, fileType);
+ tempFile = FileFactory.getCarbonFile(tempWriteFilePath, fileType);
+ byte[] buff = new byte[bufferSize];
+ dataInputStream = FileFactory.getDataInputStream(fileName, fileType);
+ // read the data
+ int read = dataInputStream.read(buff, 0, buff.length);
+ dataOutputStream = FileFactory.getDataOutputStream(tempWriteFilePath, fileType);
+ dataOutputStream.write(buff, 0, read);
+ long remaining = validDataEndOffset - read;
+ // anytime we should not cross the offset to be read
+ while (remaining > 0) {
+ if (remaining > bufferSize) {
+ buff = new byte[bufferSize];
+ } else {
+ buff = new byte[(int) remaining];
+ }
+ read = dataInputStream.read(buff, 0, buff.length);
+ dataOutputStream.write(buff, 0, read);
+ remaining = remaining - read;
+ }
+ CarbonUtil.closeStreams(dataInputStream, dataOutputStream);
+ // rename the temp file to original file
+ tempFile.renameForce(fileName);
+ fileTruncatedSuccessfully = true;
+ } catch (IOException e) {
+ LOGGER.error("Exception occured while truncating the file " + e.getMessage());
+ } finally {
+ CarbonUtil.closeStreams(dataOutputStream, dataInputStream);
+ }
+ return fileTruncatedSuccessfully;
+ }
+
+ /**
+ * This method will be used to check whether a file has been modified or not
+ *
+ * @param fileTimeStamp time to be compared with latest timestamp of file
+ * @param endOffset file length to be compared with current length of file
+ * @return
+ */
+ @Override public boolean isFileModified(long fileTimeStamp, long endOffset) {
+ boolean isFileModified = false;
+ if (getLastModifiedTime() > fileTimeStamp || getSize() > endOffset) {
+ isFileModified = true;
+ }
+ return isFileModified;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/AlluxioCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/AlluxioCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/AlluxioCarbonFile.java
new file mode 100644
index 0000000..e4ce2b6
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/AlluxioCarbonFile.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.filesystem;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+
+
+
+public class AlluxioCarbonFile extends AbstractDFSCarbonFile {
+ /**
+ * LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(AlluxioCarbonFile.class.getName());
+
+ public AlluxioCarbonFile(String filePath) {
+ super(filePath);
+ }
+
+ public AlluxioCarbonFile(Path path) {
+ super(path);
+ }
+
+ public AlluxioCarbonFile(FileStatus fileStatus) {
+ super(fileStatus);
+ }
+
+ /**
+ * @param listStatus
+ * @return
+ */
+ private CarbonFile[] getFiles(FileStatus[] listStatus) {
+ if (listStatus == null) {
+ return new CarbonFile[0];
+ }
+ CarbonFile[] files = new CarbonFile[listStatus.length];
+ for (int i = 0; i < files.length; i++) {
+ files[i] = new AlluxioCarbonFile(listStatus[i]);
+ }
+ return files;
+ }
+
+ @Override
+ public CarbonFile[] listFiles() {
+ FileStatus[] listStatus = null;
+ try {
+ if (null != fileStatus && fileStatus.isDirectory()) {
+ Path path = fileStatus.getPath();
+ listStatus = path.getFileSystem(FileFactory.getConfiguration()).listStatus(path);
+ } else {
+ return null;
+ }
+ } catch (IOException e) {
+ LOGGER.error("Exception occured: " + e.getMessage());
+ return new CarbonFile[0];
+ }
+ return getFiles(listStatus);
+ }
+
+ @Override
+ public CarbonFile[] listFiles(final CarbonFileFilter fileFilter) {
+ CarbonFile[] files = listFiles();
+ if (files != null && files.length >= 1) {
+ List<CarbonFile> fileList = new ArrayList<CarbonFile>(files.length);
+ for (int i = 0; i < files.length; i++) {
+ if (fileFilter.accept(files[i])) {
+ fileList.add(files[i]);
+ }
+ }
+ if (fileList.size() >= 1) {
+ return fileList.toArray(new CarbonFile[fileList.size()]);
+ } else {
+ return new CarbonFile[0];
+ }
+ }
+ return files;
+ }
+
+ @Override
+ public CarbonFile getParentFile() {
+ Path parent = fileStatus.getPath().getParent();
+ return null == parent ? null : new AlluxioCarbonFile(parent);
+ }
+
+ @Override
+ public boolean renameForce(String changetoName) {
+ FileSystem fs;
+ try {
+ fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
+ if (fs instanceof DistributedFileSystem) {
+ ((DistributedFileSystem) fs).rename(fileStatus.getPath(), new Path(changetoName),
+ org.apache.hadoop.fs.Options.Rename.OVERWRITE);
+ return true;
+ } else {
+ return false;
+ }
+ } catch (IOException e) {
+ LOGGER.error("Exception occured: " + e.getMessage());
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/CarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/CarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/CarbonFile.java
new file mode 100644
index 0000000..0ac4d52
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/CarbonFile.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.filesystem;
+
+public interface CarbonFile {
+
+ String getAbsolutePath();
+
+ CarbonFile[] listFiles(CarbonFileFilter fileFilter);
+
+ CarbonFile[] listFiles();
+
+ String getName();
+
+ boolean isDirectory();
+
+ boolean exists();
+
+ String getCanonicalPath();
+
+ CarbonFile getParentFile();
+
+ String getPath();
+
+ long getSize();
+
+ boolean renameTo(String changetoName);
+
+ boolean renameForce(String changetoName);
+
+ boolean delete();
+
+ boolean createNewFile();
+
+ long getLastModifiedTime();
+
+ boolean setLastModifiedTime(long timestamp);
+
+ boolean truncate(String fileName, long validDataEndOffset);
+
+ /**
+ * This method will be used to check whether a file has been modified or not
+ *
+ * @param fileTimeStamp time to be compared with latest timestamp of file
+ * @param endOffset file length to be compared with current length of file
+ * @return
+ */
+ boolean isFileModified(long fileTimeStamp, long endOffset);
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/CarbonFileFilter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/CarbonFileFilter.java b/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/CarbonFileFilter.java
new file mode 100644
index 0000000..e382f92
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/CarbonFileFilter.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.filesystem;
+
+public interface CarbonFileFilter {
+ boolean accept(CarbonFile file);
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/HDFSCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/HDFSCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/HDFSCarbonFile.java
new file mode 100644
index 0000000..a2aaca1
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/filesystem/HDFSCarbonFile.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.filesystem;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+
+public class HDFSCarbonFile extends AbstractDFSCarbonFile {
+ /**
+ * LOGGER
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(HDFSCarbonFile.class.getName());
+
+ public HDFSCarbonFile(String filePath) {
+ super(filePath);
+ }
+
+ public HDFSCarbonFile(Path path) {
+ super(path);
+ }
+
+ public HDFSCarbonFile(FileStatus fileStatus) {
+ super(fileStatus);
+ }
+
+ /**
+ * @param listStatus
+ * @return
+ */
+ private CarbonFile[] getFiles(FileStatus[] listStatus) {
+ if (listStatus == null) {
+ return new CarbonFile[0];
+ }
+ CarbonFile[] files = new CarbonFile[listStatus.length];
+ for (int i = 0; i < files.length; i++) {
+ files[i] = new HDFSCarbonFile(listStatus[i]);
+ }
+ return files;
+ }
+
+ @Override
+ public CarbonFile[] listFiles() {
+ FileStatus[] listStatus = null;
+ try {
+ if (null != fileStatus && fileStatus.isDirectory()) {
+ Path path = fileStatus.getPath();
+ listStatus = path.getFileSystem(FileFactory.getConfiguration()).listStatus(path);
+ } else {
+ return null;
+ }
+ } catch (IOException e) {
+ LOGGER.error("Exception occured: " + e.getMessage());
+ return new CarbonFile[0];
+ }
+ return getFiles(listStatus);
+ }
+
+ @Override
+ public CarbonFile[] listFiles(final CarbonFileFilter fileFilter) {
+ CarbonFile[] files = listFiles();
+ if (files != null && files.length >= 1) {
+ List<CarbonFile> fileList = new ArrayList<CarbonFile>(files.length);
+ for (int i = 0; i < files.length; i++) {
+ if (fileFilter.accept(files[i])) {
+ fileList.add(files[i]);
+ }
+ }
+ if (fileList.size() >= 1) {
+ return fileList.toArray(new CarbonFile[fileList.size()]);
+ } else {
+ return new CarbonFile[0];
+ }
+ }
+ return files;
+ }
+
+ @Override
+ public CarbonFile getParentFile() {
+ Path parent = fileStatus.getPath().getParent();
+ return null == parent ? null : new HDFSCarbonFile(parent);
+ }
+
+ @Override
+ public boolean renameForce(String changetoName) {
+ FileSystem fs;
+ try {
+ fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
+ if (fs instanceof DistributedFileSystem) {
+ ((DistributedFileSystem) fs).rename(fileStatus.getPath(), new Path(changetoName),
+ org.apache.hadoop.fs.Options.Rename.OVERWRITE);
+ return true;
+ } else {
+ return false;
+ }
+ } catch (IOException e) {
+ LOGGER.error("Exception occured: " + e.getMessage());
+ return false;
+ }
+ }
+}
\ No newline at end of file
[30/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/Compressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/Compressor.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/Compressor.java
deleted file mode 100644
index 1b90bc6..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/Compressor.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression;
-
-public interface Compressor {
-
- byte[] compressByte(byte[] unCompInput);
-
- byte[] unCompressByte(byte[] compInput);
-
- byte[] unCompressByte(byte[] compInput, int offset, int length);
-
- byte[] compressShort(short[] unCompInput);
-
- short[] unCompressShort(byte[] compInput);
-
- short[] unCompressShort(byte[] compInput, int offset, int lenght);
-
- byte[] compressInt(int[] unCompInput);
-
- int[] unCompressInt(byte[] compInput);
-
- int[] unCompressInt(byte[] compInput, int offset, int length);
-
- byte[] compressLong(long[] unCompInput);
-
- long[] unCompressLong(byte[] compInput);
-
- long[] unCompressLong(byte[] compInput, int offset, int length);
-
- byte[] compressFloat(float[] unCompInput);
-
- float[] unCompressFloat(byte[] compInput);
-
- float[] unCompressFloat(byte[] compInput, int offset, int length);
-
- byte[] compressDouble(double[] unCompInput);
-
- double[] unCompressDouble(byte[] compInput);
-
- double[] unCompressDouble(byte[] compInput, int offset, int length);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/CompressorFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/CompressorFactory.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/CompressorFactory.java
deleted file mode 100644
index 5160879..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/CompressorFactory.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.util.CarbonProperties;
-
-public class CompressorFactory {
-
- private static final CompressorFactory COMPRESSOR_FACTORY = new CompressorFactory();
-
- private final Compressor compressor;
-
- private CompressorFactory() {
- String compressorType = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.COMPRESSOR, CarbonCommonConstants.DEFAULT_COMPRESSOR);
- switch (compressorType) {
- case "snappy":
- compressor = new SnappyCompressor();
- break;
- default:
- throw new RuntimeException(
- "Invalid compressor type provided! Please provide valid compressor type");
- }
- }
-
- public static CompressorFactory getInstance() {
- return COMPRESSOR_FACTORY;
- }
-
- public Compressor getCompressor() {
- return compressor;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/MeasureMetaDataModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/MeasureMetaDataModel.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/MeasureMetaDataModel.java
deleted file mode 100644
index f207478..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/MeasureMetaDataModel.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression;
-
-public class MeasureMetaDataModel {
- /**
- * maxValue
- */
- private Object[] maxValue;
-
- /**
- * minValue
- */
- private Object[] minValue;
-
- /**
- * mantissa
- */
- private int[] mantissa;
-
- /**
- * measureCount
- */
- private int measureCount;
-
- /**
- * uniqueValue
- */
- private Object[] uniqueValue;
-
- /**
- * type
- */
- private char[] type;
-
- /**
- * dataTypeSelected
- */
- private byte[] dataTypeSelected;
-
- public MeasureMetaDataModel(Object[] minValue, Object[] maxValue, int[] mantissa,
- int measureCount, Object[] uniqueValue, char[] type, byte[] dataTypeSelected) {
- this.minValue = minValue;
- this.maxValue = maxValue;
- this.mantissa = mantissa;
- this.measureCount = measureCount;
- this.uniqueValue = uniqueValue;
- this.type = type;
- this.dataTypeSelected = dataTypeSelected;
- }
-
- /**
- * get Max value
- *
- * @return
- */
- public Object[] getMaxValue() {
- return maxValue;
- }
-
- /**
- * getMinValue
- *
- * @return
- */
- public Object[] getMinValue() {
- return minValue;
- }
-
- /**
- * getMantissa
- *
- * @return
- */
- public int[] getMantissa() {
- return mantissa;
- }
-
- /**
- * getMeasureCount
- *
- * @return
- */
- public int getMeasureCount() {
- return measureCount;
- }
-
- /**
- * getUniqueValue
- *
- * @return
- */
- public Object[] getUniqueValue() {
- return uniqueValue;
- }
-
- /**
- * @return the type
- */
- public char[] getType() {
- return type;
- }
-
- /**
- * @return the dataTypeSelected
- */
- public byte[] getDataTypeSelected() {
- return dataTypeSelected;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/ReaderCompressModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/ReaderCompressModel.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/ReaderCompressModel.java
deleted file mode 100644
index 51730a2..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/ReaderCompressModel.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression;
-
-import org.apache.carbondata.core.metadata.ValueEncoderMeta;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-
-// Used in read path for decompression preparation
-public class ReaderCompressModel {
- private ValueEncoderMeta valueEncoderMeta;
-
- private ValueCompressionUtil.DataType convertedDataType;
-
- private ValueCompressionHolder valueHolder;
-
- public void setValueEncoderMeta(ValueEncoderMeta valueEncoderMeta) {
- this.valueEncoderMeta = valueEncoderMeta;
- }
-
- public ValueCompressionUtil.DataType getConvertedDataType() {
- return convertedDataType;
- }
-
- public void setConvertedDataType(ValueCompressionUtil.DataType convertedDataType) {
- this.convertedDataType = convertedDataType;
- }
-
- public Object getMaxValue() {
- return valueEncoderMeta.getMaxValue();
- }
-
- public int getMantissa() {
- return valueEncoderMeta.getMantissa();
- }
-
- public ValueCompressionHolder getValueCompressionHolder() {
- return valueHolder;
- }
-
- public void setValueCompressionHolder(ValueCompressionHolder valueHolder) {
- this.valueHolder = valueHolder;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/SnappyCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/SnappyCompressor.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/SnappyCompressor.java
deleted file mode 100644
index 62d3b53..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/SnappyCompressor.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression;
-
-import java.io.IOException;
-import java.lang.reflect.Field;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-
-import org.xerial.snappy.Snappy;
-import org.xerial.snappy.SnappyNative;
-
-public class SnappyCompressor implements Compressor {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(SnappyCompressor.class.getName());
-
- private final SnappyNative snappyNative;
-
- public SnappyCompressor() {
- Snappy snappy = new Snappy();
- Field privateField = null;
- try {
- privateField = snappy.getClass().getDeclaredField("impl");
- } catch (NoSuchFieldException | SecurityException e) {
- throw new RuntimeException(e);
- }
- privateField.setAccessible(true);
- try {
- snappyNative = (SnappyNative) privateField.get(snappy);
- } catch (IllegalArgumentException | IllegalAccessException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override public byte[] compressByte(byte[] unCompInput) {
- try {
- return Snappy.rawCompress(unCompInput, unCompInput.length);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- return null;
- }
- }
-
- @Override public byte[] unCompressByte(byte[] compInput) {
- try {
- return Snappy.uncompress(compInput);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
- return compInput;
- }
-
- @Override public byte[] unCompressByte(byte[] compInput, int offset, int length) {
- int uncompressedLength = 0;
- byte[] data = null;
- try {
- uncompressedLength = Snappy.uncompressedLength(compInput, offset, length);
- data = new byte[uncompressedLength];
- Snappy.uncompress(compInput, offset, length, data, 0);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
- return data;
- }
-
- @Override public byte[] compressShort(short[] unCompInput) {
- try {
- return Snappy.compress(unCompInput);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- return null;
- }
- }
-
- @Override public short[] unCompressShort(byte[] compInput) {
- try {
- return Snappy.uncompressShortArray(compInput);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
- return null;
- }
-
- @Override public short[] unCompressShort(byte[] compInput, int offset, int lenght) {
- try {
- return Snappy.uncompressShortArray(compInput, offset, lenght);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
- return null;
- }
-
- @Override public byte[] compressInt(int[] unCompInput) {
- try {
- return Snappy.compress(unCompInput);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- return null;
- }
- }
-
- @Override public int[] unCompressInt(byte[] compInput) {
- try {
- return Snappy.uncompressIntArray(compInput);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
- return null;
- }
-
- @Override public int[] unCompressInt(byte[] compInput, int offset, int length) {
- try {
- return Snappy.uncompressIntArray(compInput, offset, length);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
- return null;
- }
-
- @Override public byte[] compressLong(long[] unCompInput) {
- try {
- return Snappy.compress(unCompInput);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- return null;
- }
- }
-
- @Override public long[] unCompressLong(byte[] compInput) {
- try {
- return Snappy.uncompressLongArray(compInput);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
- return null;
- }
-
- @Override public long[] unCompressLong(byte[] compInput, int offset, int length) {
- try {
- return Snappy.uncompressLongArray(compInput, offset, length);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
- return null;
- }
-
- @Override public byte[] compressFloat(float[] unCompInput) {
- try {
- return Snappy.compress(unCompInput);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- return null;
- }
- }
-
- @Override public float[] unCompressFloat(byte[] compInput) {
- try {
- return Snappy.uncompressFloatArray(compInput);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
- return null;
- }
-
- @Override public float[] unCompressFloat(byte[] compInput, int offset, int length) {
- try {
- return Snappy.uncompressFloatArray(compInput, offset, length);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
- return null;
- }
-
- @Override public byte[] compressDouble(double[] unCompInput) {
- try {
- return Snappy.compress(unCompInput);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- return null;
- }
- }
-
- @Override public double[] unCompressDouble(byte[] compInput) {
- try {
- return Snappy.uncompressDoubleArray(compInput);
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
- return null;
- }
-
- @Override public double[] unCompressDouble(byte[] compInput, int offset, int length) {
- try {
- int uncompressedLength = Snappy.uncompressedLength(compInput, offset, length);
- double[] result = new double[uncompressedLength / 8];
- snappyNative.rawUncompress(compInput, offset, length, result, 0);
- return result;
- } catch (IOException e) {
- LOGGER.error(e, e.getMessage());
- }
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/ValueCompressionHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/ValueCompressionHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/ValueCompressionHolder.java
deleted file mode 100644
index 6fee5ea..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/ValueCompressionHolder.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression;
-
-import java.math.BigDecimal;
-
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-
-
-/**
- * ValueCompressionHolder is the base class for handling
- * compression / decompression of the measure data chunk
- */
-public abstract class ValueCompressionHolder<T> {
-
- /**
- * compressedValue
- */
- protected byte[] compressedValue;
-
- /**
- * @param compressor the compressor used to decompress the data
- * @param dataType data type of the data
- * @param data compressed data
- */
- public void unCompress(Compressor compressor, DataType dataType, byte[] data,
- int offset, int length) {
- switch (dataType) {
- case DATA_BYTE:
- setValue((T)compressor.unCompressByte(data, offset, length));
- break;
- case DATA_SHORT:
- setValue((T)compressor.unCompressShort(data, offset, length));
- break;
- case DATA_INT:
- setValue((T)compressor.unCompressInt(data, offset, length));
- break;
- case DATA_LONG:
- case DATA_BIGINT:
- setValue((T)compressor.unCompressLong(data, offset, length));
- break;
- case DATA_FLOAT:
- setValue((T)compressor.unCompressFloat(data, offset, length));
- break;
- default:
- setValue((T)compressor.unCompressDouble(data, offset, length));
- break;
- }
- }
-
- /**
- * @param compressor the compressor used to compress the data
- * @param dataType data type of the data
- * @param data original data
- */
- public byte[] compress(Compressor compressor, DataType dataType, Object data) {
- switch (dataType) {
- case DATA_BYTE:
- return compressor.compressByte((byte[])data);
- case DATA_SHORT:
- return compressor.compressShort((short[])data);
- case DATA_INT:
- return compressor.compressInt((int[])data);
- case DATA_LONG:
- case DATA_BIGINT:
- return compressor.compressLong((long[])data);
- case DATA_FLOAT:
- return compressor.compressFloat((float[])data);
- case DATA_DOUBLE:
- default:
- return compressor.compressDouble((double[])data);
- }
- }
-
- public abstract void setValue(T value);
-
- public abstract T getValue();
-
- public abstract void setValueInBytes(byte[] value);
-
- public abstract void compress();
-
- public abstract void uncompress(DataType dataType, byte[] compressData, int offset,
- int length, int decimal, Object maxValueObject);
-
- public byte[] getCompressedData() { return compressedValue; }
-
- public abstract long getLongValue(int index);
-
- public abstract double getDoubleValue(int index);
-
- public abstract BigDecimal getBigDecimalValue(int index);
-
- public abstract void freeMemory();
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java
deleted file mode 100644
index 2e93123..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/WriterCompressModel.java
+++ /dev/null
@@ -1,220 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression;
-
-import org.apache.carbondata.core.util.CompressionFinder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-
-public class WriterCompressModel {
-
- /**
- * DataType[] variable.
- */
- private ValueCompressionUtil.DataType[] convertedDataType;
- /**
- * DataType[] variable.
- */
- private ValueCompressionUtil.DataType[] actualDataType;
-
- /**
- * maxValue
- */
- private Object[] maxValue;
- /**
- * minValue.
- */
- private Object[] minValue;
-
- /**
- * uniqueValue
- */
- private Object[] uniqueValue;
- /**
- * mantissa.
- */
- private int[] mantissa;
-
- /**
- * aggType
- */
- private char[] type;
-
- /**
- * dataTypeSelected
- */
- private byte[] dataTypeSelected;
- /**
- * unCompressValues.
- */
- private ValueCompressionHolder[] valueHolder;
-
- private CompressionFinder[] compressionFinders;
-
- /**
- * @return the convertedDataType
- */
- public ValueCompressionUtil.DataType[] getConvertedDataType() {
- return convertedDataType;
- }
-
- /**
- * @param convertedDataType the convertedDataType to set
- */
- public void setConvertedDataType(ValueCompressionUtil.DataType[] convertedDataType) {
- this.convertedDataType = convertedDataType;
- }
-
- /**
- * @return the actualDataType
- */
- public ValueCompressionUtil.DataType[] getActualDataType() {
- return actualDataType;
- }
-
- /**
- * @param actualDataType
- */
- public void setActualDataType(ValueCompressionUtil.DataType[] actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- /**
- * @return the maxValue
- */
- public Object[] getMaxValue() {
- return maxValue;
- }
-
- /**
- * @param maxValue the maxValue to set
- */
- public void setMaxValue(Object[] maxValue) {
- this.maxValue = maxValue;
- }
-
- /**
- * @return the mantissa
- */
- public int[] getMantissa() {
- return mantissa;
- }
-
- /**
- * @param mantissa the mantissa to set
- */
- public void setMantissa(int[] mantissa) {
- this.mantissa = mantissa;
- }
-
- /**
- * getUnCompressValues().
- *
- * @return the unCompressValues
- */
- public ValueCompressionHolder[] getValueCompressionHolder() {
- return valueHolder;
- }
-
- /**
- * @param valueHolder set the ValueCompressionHolder
- */
- public void setValueCompressionHolder(ValueCompressionHolder[] valueHolder) {
- this.valueHolder = valueHolder;
- }
-
- /**
- * getMinValue
- *
- * @return
- */
- public Object[] getMinValue() {
- return minValue;
- }
-
- /**
- * setMinValue.
- *
- * @param minValue
- */
- public void setMinValue(Object[] minValue) {
- this.minValue = minValue;
- }
-
- /**
- * @return the aggType
- */
- public char[] getType() {
- return type;
- }
-
- /**
- * @param type the type to set
- */
- public void setType(char[] type) {
- this.type = type;
- }
-
- /**
- * @return the dataTypeSelected
- */
- public byte[] getDataTypeSelected() {
- return dataTypeSelected;
- }
-
- /**
- * @param dataTypeSelected the dataTypeSelected to set
- */
- public void setDataTypeSelected(byte[] dataTypeSelected) {
- this.dataTypeSelected = dataTypeSelected;
- }
-
- /**
- * getUniqueValue
- *
- * @return
- */
- public Object[] getUniqueValue() {
- return uniqueValue;
- }
-
- /**
- * setUniqueValue
- *
- * @param uniqueValue
- */
- public void setUniqueValue(Object[] uniqueValue) {
- this.uniqueValue = uniqueValue;
- }
-
- public void setCompressionFinders(CompressionFinder[] compressionFinders) {
- this.compressionFinders = compressionFinders;
- }
-
- public CompressionFinder[] getCompressionFinders() {
- return this.compressionFinders;
- }
-
- /**
- * @return the compType
- */
- public ValueCompressionUtil.COMPRESSION_TYPE getCompType(int index) {
- return this.compressionFinders[index].getCompType();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinByte.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinByte.java
deleted file mode 100644
index bd1c643..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinByte.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
-3 * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.decimal;
-
-import java.math.BigDecimal;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionMaxMinByte extends ValueCompressionHolder<byte[]> {
-
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionMaxMinByte.class.getName());
-
- /**
- * compressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- /**
- * value.
- */
- protected byte[] value;
-
- private MeasureDataChunkStore<byte[]> measureChunkStore;
-
- /**
- * actual data type
- */
- protected DataType actualDataType;
-
- private double maxValue;
-
- public CompressionMaxMinByte(DataType actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- @Override public byte[] getValue() {return this.value; }
-
- @Override public void setValue(byte[] value) {
- this.value = value;
- }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_BYTE, value);
- }
-
- @Override
- public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
- int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, compressedData, offset, length);
- setUncompressedValues(value, maxValueObject);
-
- }
-
- @Override public void setValueInBytes(byte[] value) {
- this.value = value;
- }
-
- @Override public long getLongValue(int index) {
- byte byteValue = measureChunkStore.getByte(index);
- return (long) (maxValue - byteValue);
- }
-
- @Override public double getDoubleValue(int index) {
- byte byteValue = measureChunkStore.getByte(index);
- return (maxValue - byteValue);
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionMaxMinByte");
- }
-
- private void setUncompressedValues(byte[] data, Object maxValueObject) {
- this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_BYTE, data.length);
- this.measureChunkStore.putData(data);
- if (maxValueObject instanceof Long) {
- this.maxValue = (long)maxValueObject;
- } else {
- this.maxValue = (double) maxValueObject;
- }
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinDefault.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinDefault.java
deleted file mode 100644
index 1c38d11..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinDefault.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.decimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-
-public class CompressionMaxMinDefault extends ValueCompressionHolder<double[]> {
-
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionMaxMinDefault.class.getName());
-
- /**
- * compressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- /**
- * value.
- */
- private double[] value;
-
- private MeasureDataChunkStore<double[]> measureChunkStore;
-
- /**
- * actual data type
- */
- private DataType actualDataType;
-
- private double maxValue;
-
- public CompressionMaxMinDefault(DataType actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- @Override public void setValue(double[] value) {
- this.value = value;
- }
-
- @Override public double[] getValue() {return this.value; }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_DOUBLE, value);
- }
-
- @Override public void uncompress(DataType dataType, byte[] compressedData,
- int offset, int length, int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, compressedData, offset, length);
- setUncompressedValues(value, maxValueObject);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToDoubleArray(buffer, value.length);
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long value is not defined for CompressionMaxMinDefault");
- }
-
- @Override public double getDoubleValue(int index) {
- double doubleValue = measureChunkStore.getDouble(index);
- return maxValue - doubleValue;
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionMaxMinDefault");
- }
-
- private void setUncompressedValues(double[] data, Object maxValueObject) {
- this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_DOUBLE, data.length);
- this.measureChunkStore.putData(data);
- if (maxValueObject instanceof Long) {
- this.maxValue = (long) maxValueObject;
- } else {
- this.maxValue = (double) maxValueObject;
- }
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinInt.java
deleted file mode 100644
index 940b637..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinInt.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.decimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionMaxMinInt extends ValueCompressionHolder<int[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionMaxMinInt.class.getName());
-
- /**
- * compressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- private MeasureDataChunkStore<int[]> measureChunkStore;
-
- /**
- * value.
- */
- private int[] value;
-
- private DataType actualDataType;
-
- private double maxValue;
-
- public CompressionMaxMinInt(DataType actualType) {
- this.actualDataType = actualType;
- }
-
- @Override public void setValue(int[] value) {
- this.value = value;
- }
-
- @Override public int[] getValue() { return this.value; }
-
- @Override
- public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
- int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, compressedData, offset, length);
- setUncompressedValues(value, maxValueObject);
- }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_INT, value);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToIntArray(buffer, value.length);
- }
-
- @Override public long getLongValue(int index) {
- int intValue = measureChunkStore.getInt(index);
- return (long) (maxValue - intValue);
- }
-
- @Override public double getDoubleValue(int index) {
- int intValue = measureChunkStore.getInt(index);
- return maxValue - intValue;
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionMaxMinInt");
- }
-
- private void setUncompressedValues(int[] data, Object maxValueObject) {
- this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, data.length);
- this.measureChunkStore.putData(data);
- if (maxValueObject instanceof Long) {
- this.maxValue = (long) maxValueObject;
- } else {
- this.maxValue = (double) maxValueObject;
- }
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinLong.java
deleted file mode 100644
index 846d93e..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinLong.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.decimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionMaxMinLong extends ValueCompressionHolder<long[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionMaxMinLong.class.getName());
- /**
- * compressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- private MeasureDataChunkStore<long[]> measureChunkStore;
-
- /**
- * value.
- */
- protected long[] value;
-
- protected DataType actualDataType;
-
- private double maxValue;
-
- public CompressionMaxMinLong(DataType actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_LONG, value);
- }
-
- @Override public void setValue(long[] value) {
- this.value = value;
-
- }
-
- @Override
- public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
- int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, compressedData, offset, length);
- setUncompressValues(value, maxValueObject);
- }
-
- @Override public long[] getValue() {return this.value; }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToLongArray(buffer, value.length);
- }
-
- @Override public long getLongValue(int index) {
- long longValue = measureChunkStore.getLong(index);
- return (long) maxValue - longValue;
- }
-
- @Override public double getDoubleValue(int index) {
- long longValue = measureChunkStore.getLong(index);
- return maxValue - longValue;
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionMaxMinLong");
- }
-
- private void setUncompressValues(long[] data, Object maxValueObject) {
- this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_LONG, data.length);
- this.measureChunkStore.putData(data);
- if (maxValueObject instanceof Long) {
- this.maxValue = (long) maxValueObject;
- } else {
- this.maxValue = (double) maxValueObject;
- }
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinShort.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinShort.java
deleted file mode 100644
index f12ba69..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/decimal/CompressionMaxMinShort.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.decimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionMaxMinShort extends ValueCompressionHolder<short[]> {
-
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionMaxMinShort.class.getName());
-
- /**
- * shortCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- private MeasureDataChunkStore<short[]> measureChunkStore;
-
- /**
- * value.
- */
- private short[] value;
-
- private DataType actualDataType;
-
- private double maxValue;
-
- public CompressionMaxMinShort(DataType actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- @Override public void setValue(short[] value) {
- this.value = value;
-
- }
-
- @Override
- public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
- int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, compressedData, offset, length);
- setUncompressedValues(value, maxValueObject);
- }
-
- @Override public short[] getValue() {return this.value; }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_SHORT, value);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToShortArray(buffer, value.length);
- }
-
- @Override public long getLongValue(int index) {
- short shortValue = measureChunkStore.getShort(index);
- return (long) maxValue - shortValue;
- }
-
- @Override public double getDoubleValue(int index) {
- short shortValue = measureChunkStore.getShort(index);
- return maxValue - shortValue;
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionMaxMinShort");
- }
-
- private void setUncompressedValues(short[] data, Object maxValueObject) {
- this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_SHORT, data.length);
- this.measureChunkStore.putData(data);
- if (maxValueObject instanceof Long) {
- this.maxValue = (long) maxValueObject;
- } else {
- this.maxValue = (double) maxValueObject;
- }
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalByte.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalByte.java
deleted file mode 100644
index 3f0a99c..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalByte.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.nondecimal;
-
-import java.math.BigDecimal;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNonDecimalByte extends ValueCompressionHolder<byte[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNonDecimalByte.class.getName());
-
- /**
- * compressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- /**
- * value.
- */
- private byte[] value;
-
- private MeasureDataChunkStore<byte[]> measureChunkStore;
-
- private double divisionFactory;
-
- @Override public void setValue(byte[] value) {
- this.value = value;
- }
-
- @Override public byte[] getValue() {return this.value; }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_BYTE, value);
- }
-
- @Override
- public void uncompress(DataType dataType, byte[] compressedData,
- int offset, int length, int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, compressedData, offset, length);
- setUncompressedValues(value, decimalPlaces);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- this.value = value;
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long value is not defined for CompressionNonDecimalByte");
- }
-
- @Override public double getDoubleValue(int index) {
- return (measureChunkStore.getByte(index) / this.divisionFactory);
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionNonDecimalByte");
- }
-
- private void setUncompressedValues(byte[] data, int decimalPlaces) {
- this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_BYTE, data.length);
- this.measureChunkStore.putData(data);
- this.divisionFactory = Math.pow(10, decimalPlaces);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalDefault.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalDefault.java
deleted file mode 100644
index 48e4b5e..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalDefault.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.nondecimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNonDecimalDefault extends ValueCompressionHolder<double[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNonDecimalDefault.class.getName());
- /**
- * doubleCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
- /**
- * value.
- */
- private double[] value;
-
- private MeasureDataChunkStore<double[]> measureChunkStore;
-
- private double divisionFactory;
-
- @Override
- public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
- int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, compressedData, offset, length);
- setUncompressedValues(value, decimalPlaces);
- }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_DOUBLE, value);
- }
-
- @Override public void setValue(double[] value) {
- this.value = value;
-
- }
-
- @Override public double[] getValue() {return this.value; }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToDoubleArray(buffer, value.length);
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long value is not defined for CompressionNonDecimalDefault");
- }
-
- @Override public double getDoubleValue(int index) {
- return (measureChunkStore.getDouble(index) / divisionFactory);
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionNonDecimalDefault");
- }
-
- private void setUncompressedValues(double[] data, int decimalPlaces) {
- this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_DOUBLE, data.length);
- this.measureChunkStore.putData(data);
- this.divisionFactory = Math.pow(10, decimalPlaces);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalInt.java
deleted file mode 100644
index 1ae5595..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalInt.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.nondecimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNonDecimalInt extends ValueCompressionHolder<int[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNonDecimalInt.class.getName());
- /**
- * intCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
- /**
- * value.
- */
- private int[] value;
-
- private MeasureDataChunkStore<int[]> measureChunkStore;
-
- private double divisionFactory;
-
- @Override public void setValue(int[] value) {
- this.value = value;
- }
-
- @Override public int[] getValue() { return this.value; }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_INT, value);
- }
-
- @Override public void setValueInBytes(byte[] bytesArr) {
- ByteBuffer buffer = ByteBuffer.wrap(bytesArr);
- this.value = ValueCompressionUtil.convertToIntArray(buffer, bytesArr.length);
- }
-
- @Override
- public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
- int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, compressedData, offset, length);
- setUncompressedValues(value, decimalPlaces);
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long value is not defined for CompressionNonDecimalInt");
- }
-
- @Override public double getDoubleValue(int index) {
- return (measureChunkStore.getInt(index) / this.divisionFactory);
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionNonDecmialInt");
- }
-
- private void setUncompressedValues(int[] data, int decimalPlaces) {
- this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, data.length);
- this.measureChunkStore.putData(data);
- this.divisionFactory = Math.pow(10, decimalPlaces);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalLong.java
deleted file mode 100644
index 8152647..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalLong.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.nondecimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNonDecimalLong extends ValueCompressionHolder<long[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNonDecimalLong.class.getName());
-
- /**
- * longCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- /**
- * value.
- */
- private long[] value;
-
- private MeasureDataChunkStore<long[]> measureChunkStore;
-
- private double divisionFactory;
-
- @Override public void setValue(long[] value) {
- this.value = value;
- }
-
- @Override public long[] getValue() { return this.value; }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_LONG, value);
- }
-
- @Override public void uncompress(DataType dataType, byte[] compressedData, int offset,
- int length, int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, compressedData, offset, length);
- setUncompressedValues(value, decimalPlaces);
- }
-
- @Override public void setValueInBytes(byte[] bytes) {
- ByteBuffer buffer = ByteBuffer.wrap(bytes);
- this.value = ValueCompressionUtil.convertToLongArray(buffer, bytes.length);
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long value is not defined for CompressionNonDecimalLong");
- }
-
- @Override public double getDoubleValue(int index) {
- return (measureChunkStore.getLong(index) / this.divisionFactory);
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionNonDecimalLong");
- }
-
- private void setUncompressedValues(long[] data, int decimalPlaces) {
- this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_LONG, data.length);
- this.measureChunkStore.putData(data);
- this.divisionFactory = Math.pow(10, decimalPlaces);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalMaxMinByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalMaxMinByte.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalMaxMinByte.java
deleted file mode 100644
index 913c3bd..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalMaxMinByte.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.nondecimal;
-
-import java.math.BigDecimal;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNonDecimalMaxMinByte extends ValueCompressionHolder<byte[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNonDecimalMaxMinByte.class.getName());
-
- /**
- * compressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- /**
- * value.
- */
- private byte[] value;
-
- private MeasureDataChunkStore<byte[]> measureChunkStore;
-
- private BigDecimal maxValue;
-
- private double divisionFactor;
-
- @Override
- public void uncompress(DataType dataType, byte[] compressedData,
- int offset, int length, int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, compressedData, offset, length);
- setUncompressedValues(value, decimalPlaces, maxValueObject );
- }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_BYTE, value);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- this.value = value;
- }
-
- @Override public void setValue(byte[] value) {
- this.value = value;
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long value is not defined for CompressionNonDecimalMaxMinByte");
- }
-
- @Override public double getDoubleValue(int index) {
- byte byteValue = measureChunkStore.getByte(index);
- BigDecimal diff = BigDecimal.valueOf(byteValue / this.divisionFactor);
- return maxValue.subtract(diff).doubleValue();
-
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionNonDecimalMaxMinByte");
- }
-
- private void setUncompressedValues(byte[] data, int decimalPlaces, Object maxValueObject) {
- this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_BYTE, data.length);
- this.measureChunkStore.putData(data);
- this.maxValue = BigDecimal.valueOf((double) maxValueObject);
- this.divisionFactor = Math.pow(10, decimalPlaces);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-
- @Override public byte[] getValue() { return this.value; }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java
deleted file mode 100644
index a52f6de..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.nondecimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNonDecimalMaxMinDefault extends ValueCompressionHolder<double[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNonDecimalMaxMinDefault.class.getName());
-
- /**
- * doubleCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- /**
- * value.
- */
- private double[] value;
-
- private MeasureDataChunkStore<double[]> measureChunkStore;
-
- private BigDecimal maxValue;
-
- private double divisionFactor;
-
- @Override public void setValue(double[] value) {
- this.value = value;
- }
-
- @Override public double[] getValue() { return this.value; }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_DOUBLE, value);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToDoubleArray(buffer, value.length);
- }
-
- @Override
- public void uncompress(DataType dataType, byte[] compressedData,
- int offset, int length, int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor,dataType,compressedData, offset, length);
- setUncompressedValues(value, decimalPlaces, maxValueObject);
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long value is not defined for CompressionNonDecimalMaxMinDefault");
- }
-
- @Override public double getDoubleValue(int index) {
- double doubleValue = measureChunkStore.getDouble(index);
- BigDecimal diff = BigDecimal.valueOf(doubleValue / this.divisionFactor);
- return maxValue.subtract(diff).doubleValue();
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionNonDecimalMaxMinDefault");
- }
-
- private void setUncompressedValues(double[] data, int decimalPlaces, Object maxValueObject) {
- this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_DOUBLE, data.length);
- this.measureChunkStore.putData(data);
- this.maxValue = BigDecimal.valueOf((double) maxValueObject);
- this.divisionFactor = Math.pow(10, decimalPlaces);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalMaxMinInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalMaxMinInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalMaxMinInt.java
deleted file mode 100644
index cbb63db..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalMaxMinInt.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.nondecimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNonDecimalMaxMinInt extends ValueCompressionHolder<int[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNonDecimalMaxMinInt.class.getName());
- /**
- * intCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
- /**
- * value.
- */
- private int[] value;
-
- private MeasureDataChunkStore<int[]> measureChunkStore;
-
- private double divisionFactor;
-
- private BigDecimal maxValue;
-
- @Override public void setValue(int[] value) {
- this.value = value;
- }
-
- @Override public int[] getValue() {return this.value; }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_INT, value);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToIntArray(buffer, value.length);
- }
-
- @Override
- public void uncompress(DataType dataType, byte[] compressedData,
- int offset, int length, int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, compressedData, offset, length);
- setUncompressedValues(value, decimalPlaces, maxValueObject);
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long value is not defined for CompressionNonDecimalMaxMinInt");
- }
-
- @Override public double getDoubleValue(int index) {
- int intValue = measureChunkStore.getInt(index);
- BigDecimal diff = BigDecimal.valueOf(intValue / this.divisionFactor);
- return maxValue.subtract(diff).doubleValue();
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionNonDecimalMaxMinInt");
- }
-
- private void setUncompressedValues(int[] data, int decimalPlaces, Object maxValueObject) {
- this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, data.length);
- this.measureChunkStore.putData(data);
- this.maxValue = BigDecimal.valueOf((double) maxValueObject);
- this.divisionFactor = Math.pow(10, decimalPlaces);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalMaxMinLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalMaxMinLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalMaxMinLong.java
deleted file mode 100644
index 00aa2ae..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalMaxMinLong.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.nondecimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNonDecimalMaxMinLong extends ValueCompressionHolder<long[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNonDecimalMaxMinLong.class.getName());
-
- /**
- * longCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
- /**
- * value.
- */
- private long[] value;
-
- private MeasureDataChunkStore<long[]> measureChunkStore;
-
- private BigDecimal maxValue;
-
- private double divisionFactor;
-
- @Override public void setValue(long[] value) {
- this.value = value;
-
- }
-
- @Override public long[] getValue() { return this.value; }
-
- @Override
- public void uncompress(DataType dataType, byte[] compressedData,
- int offset, int length, int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, compressedData, offset, length);
- setUncompressedValues(value, decimalPlaces, maxValueObject);
- }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_LONG, value);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buff = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToLongArray(buff, value.length);
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long value is not defined for CompressionNonDecimalMaxMinLong");
- }
-
- @Override public double getDoubleValue(int index) {
- long longValue = measureChunkStore.getLong(index);
- BigDecimal diff = BigDecimal.valueOf(longValue / this.divisionFactor);
- return maxValue.subtract(diff).doubleValue();
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException("Big decimal value is not supported");
- }
-
- private void setUncompressedValues(long[] data, int decimalPlaces, Object maxValueObject) {
- this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_LONG, data.length);
- this.measureChunkStore.putData(data);
- this.maxValue = BigDecimal.valueOf((double) maxValueObject);
- this.divisionFactor = Math.pow(10, decimalPlaces);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
[18/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java
deleted file mode 100644
index 3c45d2d..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java
+++ /dev/null
@@ -1,483 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.executor.impl;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.common.logging.impl.StandardLogService;
-import org.apache.carbondata.core.cache.CacheProvider;
-import org.apache.carbondata.core.cache.CacheType;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.datastore.BlockIndexStore;
-import org.apache.carbondata.core.datastore.IndexKey;
-import org.apache.carbondata.core.datastore.block.AbstractIndex;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.datastore.block.TableBlockInfo;
-import org.apache.carbondata.core.datastore.block.TableBlockUniqueIdentifier;
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.core.metadata.Encoding;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-import org.apache.carbondata.core.path.CarbonStorePath;
-import org.apache.carbondata.core.stats.QueryStatistic;
-import org.apache.carbondata.core.stats.QueryStatisticsConstants;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
-import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.scan.executor.QueryExecutor;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
-import org.apache.carbondata.scan.executor.infos.AggregatorInfo;
-import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.scan.executor.util.QueryUtil;
-import org.apache.carbondata.scan.executor.util.RestructureUtil;
-import org.apache.carbondata.scan.filter.FilterUtil;
-import org.apache.carbondata.scan.model.QueryDimension;
-import org.apache.carbondata.scan.model.QueryMeasure;
-import org.apache.carbondata.scan.model.QueryModel;
-
-import org.apache.commons.lang3.ArrayUtils;
-
-/**
- * This class provides a skeletal implementation of the {@link QueryExecutor}
- * interface to minimize the effort required to implement this interface. This
- * will be used to prepare all the properties required for query execution
- */
-public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(AbstractQueryExecutor.class.getName());
- /**
- * holder for query properties which will be used to execute the query
- */
- protected QueryExecutorProperties queryProperties;
-
- public AbstractQueryExecutor() {
- queryProperties = new QueryExecutorProperties();
- }
-
- /**
- * Below method will be used to fill the executor properties based on query
- * model it will parse the query model and get the detail and fill it in
- * query properties
- *
- * @param queryModel
- */
- protected void initQuery(QueryModel queryModel) throws IOException {
- StandardLogService.setThreadName(StandardLogService.getPartitionID(
- queryModel.getAbsoluteTableIdentifier().getCarbonTableIdentifier().getTableName()),
- queryModel.getQueryId());
- LOGGER.info("Query will be executed on table: " + queryModel.getAbsoluteTableIdentifier()
- .getCarbonTableIdentifier().getTableName());
- // add executor service for query execution
- queryProperties.executorService = Executors.newFixedThreadPool(1);
- // Initializing statistics list to record the query statistics
- // creating copy on write to handle concurrent scenario
- queryProperties.queryStatisticsRecorder =
- CarbonTimeStatisticsFactory.createExecutorRecorder(queryModel.getQueryId());
- queryModel.setStatisticsRecorder(queryProperties.queryStatisticsRecorder);
- QueryUtil.resolveQueryModel(queryModel);
- QueryStatistic queryStatistic = new QueryStatistic();
- // sort the block info
- // so block will be loaded in sorted order this will be required for
- // query execution
- Collections.sort(queryModel.getTableBlockInfos());
- // get the table blocks
- CacheProvider cacheProvider = CacheProvider.getInstance();
- BlockIndexStore<TableBlockUniqueIdentifier, AbstractIndex> cache =
- (BlockIndexStore) cacheProvider
- .createCache(CacheType.EXECUTOR_BTREE, queryModel.getTable().getStorePath());
- // remove the invalid table blocks, block which is deleted or compacted
- cache.removeTableBlocks(queryModel.getInvalidSegmentIds(),
- queryModel.getAbsoluteTableIdentifier());
- List<TableBlockUniqueIdentifier> tableBlockUniqueIdentifiers =
- prepareTableBlockUniqueIdentifier(queryModel.getTableBlockInfos(),
- queryModel.getAbsoluteTableIdentifier());
- cache.removeTableBlocksIfHorizontalCompactionDone(queryModel);
- queryProperties.dataBlocks = cache.getAll(tableBlockUniqueIdentifiers);
- queryStatistic
- .addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_EXECUTOR, System.currentTimeMillis());
- queryProperties.queryStatisticsRecorder.recordStatistics(queryStatistic);
- //
- // // updating the restructuring infos for the query
- queryProperties.keyStructureInfo = getKeyStructureInfo(queryModel,
- queryProperties.dataBlocks.get(queryProperties.dataBlocks.size() - 1).getSegmentProperties()
- .getDimensionKeyGenerator());
-
- // calculating the total number of aggeragted columns
- int aggTypeCount = queryModel.getQueryMeasures().size();
-
- int currentIndex = 0;
- DataType[] dataTypes = new DataType[aggTypeCount];
-
- for (QueryMeasure carbonMeasure : queryModel.getQueryMeasures()) {
- // adding the data type and aggregation type of all the measure this
- // can be used
- // to select the aggregator
- dataTypes[currentIndex] = carbonMeasure.getMeasure().getDataType();
- currentIndex++;
- }
- queryProperties.measureDataTypes = dataTypes;
- // as aggregation will be executed in following order
- // 1.aggregate dimension expression
- // 2. expression
- // 3. query measure
- // so calculating the index of the expression start index
- // and measure column start index
- queryProperties.filterMeasures = new HashSet<>();
- queryProperties.complexFilterDimension = new HashSet<>();
- QueryUtil.getAllFilterDimensions(queryModel.getFilterExpressionResolverTree(),
- queryProperties.complexFilterDimension, queryProperties.filterMeasures);
-
- queryStatistic = new QueryStatistic();
- // dictionary column unique column id to dictionary mapping
- // which will be used to get column actual data
- queryProperties.columnToDictionayMapping = QueryUtil
- .getDimensionDictionaryDetail(queryModel.getQueryDimension(),
- queryProperties.complexFilterDimension, queryModel.getAbsoluteTableIdentifier());
- queryStatistic
- .addStatistics(QueryStatisticsConstants.LOAD_DICTIONARY, System.currentTimeMillis());
- queryProperties.queryStatisticsRecorder.recordStatistics(queryStatistic);
- queryModel.setColumnToDictionaryMapping(queryProperties.columnToDictionayMapping);
- }
-
- private List<TableBlockUniqueIdentifier> prepareTableBlockUniqueIdentifier(
- List<TableBlockInfo> tableBlockInfos, AbsoluteTableIdentifier absoluteTableIdentifier) {
- List<TableBlockUniqueIdentifier> tableBlockUniqueIdentifiers =
- new ArrayList<>(tableBlockInfos.size());
- for (TableBlockInfo blockInfo : tableBlockInfos) {
- tableBlockUniqueIdentifiers
- .add(new TableBlockUniqueIdentifier(absoluteTableIdentifier, blockInfo));
- }
- return tableBlockUniqueIdentifiers;
- }
-
- /**
- * Below method will be used to get the key structure info for the query
- *
- * @param queryModel query model
- * @param keyGenerator
- * @return key structure info
- */
- private KeyStructureInfo getKeyStructureInfo(QueryModel queryModel, KeyGenerator keyGenerator) {
- // getting the masked byte range for dictionary column
- int[] maskByteRanges =
- QueryUtil.getMaskedByteRange(queryModel.getQueryDimension(), keyGenerator);
-
- // max key for the dictionary dimension present in the query
- byte[] maxKey = null;
- try {
- // getting the max key which will be used to masked and get the
- // masked key
- maxKey = QueryUtil.getMaxKeyBasedOnDimensions(queryModel.getQueryDimension(), keyGenerator);
- } catch (KeyGenException e) {
- LOGGER.error(e, "problem while getting the max key");
- }
-
- KeyStructureInfo restructureInfos = new KeyStructureInfo();
- restructureInfos.setKeyGenerator(keyGenerator);
- restructureInfos.setMaskByteRanges(maskByteRanges);
- restructureInfos.setMaxKey(maxKey);
- return restructureInfos;
- }
-
- protected List<BlockExecutionInfo> getBlockExecutionInfos(QueryModel queryModel)
- throws IOException, QueryExecutionException {
- initQuery(queryModel);
- List<BlockExecutionInfo> blockExecutionInfoList = new ArrayList<BlockExecutionInfo>();
- // fill all the block execution infos for all the blocks selected in
- // query
- // and query will be executed based on that infos
- for (int i = 0; i < queryProperties.dataBlocks.size(); i++) {
- blockExecutionInfoList.add(
- getBlockExecutionInfoForBlock(queryModel, queryProperties.dataBlocks.get(i),
- queryModel.getTableBlockInfos().get(i).getBlockletInfos().getStartBlockletNumber(),
- queryModel.getTableBlockInfos().get(i).getBlockletInfos().getNumberOfBlockletToScan(),
- queryModel.getTableBlockInfos().get(i).getFilePath()));
- }
- if (null != queryModel.getStatisticsRecorder()) {
- QueryStatistic queryStatistic = new QueryStatistic();
- queryStatistic.addCountStatistic(QueryStatisticsConstants.SCAN_BLOCKS_NUM,
- blockExecutionInfoList.size());
- queryModel.getStatisticsRecorder().recordStatistics(queryStatistic);
- }
- return blockExecutionInfoList;
- }
-
- /**
- * Below method will be used to get the block execution info which is
- * required to execute any block based on query model
- *
- * @param queryModel query model from user query
- * @param blockIndex block index
- * @return block execution info
- * @throws QueryExecutionException any failure during block info creation
- */
- protected BlockExecutionInfo getBlockExecutionInfoForBlock(QueryModel queryModel,
- AbstractIndex blockIndex, int startBlockletIndex, int numberOfBlockletToScan, String filePath)
- throws QueryExecutionException {
- BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
- SegmentProperties segmentProperties = blockIndex.getSegmentProperties();
- List<CarbonDimension> tableBlockDimensions = segmentProperties.getDimensions();
- KeyGenerator blockKeyGenerator = segmentProperties.getDimensionKeyGenerator();
-
- // below is to get only those dimension in query which is present in the
- // table block
- List<QueryDimension> updatedQueryDimension = RestructureUtil
- .getUpdatedQueryDimension(queryModel.getQueryDimension(), tableBlockDimensions,
- segmentProperties.getComplexDimensions());
- // TODO add complex dimension children
- int[] maskByteRangesForBlock =
- QueryUtil.getMaskedByteRange(updatedQueryDimension, blockKeyGenerator);
- int[] maksedByte =
- QueryUtil.getMaskedByte(blockKeyGenerator.getKeySizeInBytes(), maskByteRangesForBlock);
- int tableFactPathLength = CarbonStorePath
- .getCarbonTablePath(queryModel.getAbsoluteTableIdentifier().getStorePath(),
- queryModel.getAbsoluteTableIdentifier().getCarbonTableIdentifier()).getFactDir()
- .length() + 1;
- blockExecutionInfo.setBlockId(filePath.substring(tableFactPathLength));
- blockExecutionInfo.setStartBlockletIndex(startBlockletIndex);
- blockExecutionInfo.setNumberOfBlockletToScan(numberOfBlockletToScan);
- blockExecutionInfo.setQueryDimensions(
- updatedQueryDimension.toArray(new QueryDimension[updatedQueryDimension.size()]));
- blockExecutionInfo.setQueryMeasures(queryModel.getQueryMeasures()
- .toArray(new QueryMeasure[queryModel.getQueryMeasures().size()]));
- blockExecutionInfo.setDataBlock(blockIndex);
- blockExecutionInfo.setBlockKeyGenerator(blockKeyGenerator);
- // adding aggregation info for query
- blockExecutionInfo.setAggregatorInfo(getAggregatorInfoForBlock(queryModel, blockIndex));
- // setting whether raw record query or not
- blockExecutionInfo.setRawRecordDetailQuery(queryModel.isForcedDetailRawQuery());
- // setting the masked byte of the block which will be
- // used to update the unpack the older block keys
- blockExecutionInfo.setMaskedByteForBlock(maksedByte);
- // total number dimension
- blockExecutionInfo
- .setTotalNumberDimensionBlock(segmentProperties.getDimensionOrdinalToBlockMapping().size());
- blockExecutionInfo
- .setTotalNumberOfMeasureBlock(segmentProperties.getMeasuresOrdinalToBlockMapping().size());
- blockExecutionInfo.setAbsoluteTableIdentifier(queryModel.getAbsoluteTableIdentifier());
- blockExecutionInfo.setComplexDimensionInfoMap(QueryUtil
- .getComplexDimensionsMap(updatedQueryDimension,
- segmentProperties.getDimensionOrdinalToBlockMapping(),
- segmentProperties.getEachComplexDimColumnValueSize(),
- queryProperties.columnToDictionayMapping, queryProperties.complexFilterDimension));
- // to check whether older block key update is required or not
- blockExecutionInfo.setFixedKeyUpdateRequired(
- !blockKeyGenerator.equals(queryProperties.keyStructureInfo.getKeyGenerator()));
- IndexKey startIndexKey = null;
- IndexKey endIndexKey = null;
- if (null != queryModel.getFilterExpressionResolverTree()) {
- // loading the filter executer tree for filter evaluation
- blockExecutionInfo.setFilterExecuterTree(FilterUtil
- .getFilterExecuterTree(queryModel.getFilterExpressionResolverTree(), segmentProperties,
- blockExecutionInfo.getComlexDimensionInfoMap()));
- List<IndexKey> listOfStartEndKeys = new ArrayList<IndexKey>(2);
- FilterUtil.traverseResolverTreeAndGetStartAndEndKey(segmentProperties,
- queryModel.getFilterExpressionResolverTree(), listOfStartEndKeys);
- startIndexKey = listOfStartEndKeys.get(0);
- endIndexKey = listOfStartEndKeys.get(1);
- } else {
- try {
- startIndexKey = FilterUtil.prepareDefaultStartIndexKey(segmentProperties);
- endIndexKey = FilterUtil.prepareDefaultEndIndexKey(segmentProperties);
- } catch (KeyGenException e) {
- throw new QueryExecutionException(e);
- }
- }
- //setting the start index key of the block node
- blockExecutionInfo.setStartKey(startIndexKey);
- //setting the end index key of the block node
- blockExecutionInfo.setEndKey(endIndexKey);
- // expression dimensions
- List<CarbonDimension> expressionDimensions =
- new ArrayList<CarbonDimension>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- // expression measure
- List<CarbonMeasure> expressionMeasures =
- new ArrayList<CarbonMeasure>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- // setting all the dimension chunk indexes to be read from file
- int numberOfElementToConsider = 0;
- // list of dimensions to be projected
- List<Integer> allProjectionListDimensionIdexes = new ArrayList<>();
- int[] dimensionsBlockIndexes = QueryUtil.getDimensionsBlockIndexes(updatedQueryDimension,
- segmentProperties.getDimensionOrdinalToBlockMapping(), expressionDimensions,
- queryProperties.complexFilterDimension, allProjectionListDimensionIdexes);
- if (dimensionsBlockIndexes.length > 0) {
- numberOfElementToConsider = dimensionsBlockIndexes[dimensionsBlockIndexes.length - 1]
- == segmentProperties.getBlockTodimensionOrdinalMapping().size() - 1 ?
- dimensionsBlockIndexes.length - 1 :
- dimensionsBlockIndexes.length;
- blockExecutionInfo.setAllSelectedDimensionBlocksIndexes(CarbonUtil
- .getRangeIndex(dimensionsBlockIndexes, numberOfElementToConsider,
- CarbonCommonConstants.NUMBER_OF_COLUMN_READ_IN_IO));
- } else {
- blockExecutionInfo.setAllSelectedDimensionBlocksIndexes(new int[0][0]);
- }
-
- int[] measureBlockIndexes = QueryUtil
- .getMeasureBlockIndexes(queryModel.getQueryMeasures(), expressionMeasures,
- segmentProperties.getMeasuresOrdinalToBlockMapping(), queryProperties.filterMeasures);
- if (measureBlockIndexes.length > 0) {
-
- numberOfElementToConsider = measureBlockIndexes[measureBlockIndexes.length - 1]
- == segmentProperties.getMeasures().size() - 1 ?
- measureBlockIndexes.length - 1 :
- measureBlockIndexes.length;
- // setting all the measure chunk indexes to be read from file
- blockExecutionInfo.setAllSelectedMeasureBlocksIndexes(CarbonUtil
- .getRangeIndex(measureBlockIndexes, numberOfElementToConsider,
- CarbonCommonConstants.NUMBER_OF_COLUMN_READ_IN_IO));
- } else {
- blockExecutionInfo.setAllSelectedMeasureBlocksIndexes(new int[0][0]);
- }
- // setting the key structure info which will be required
- // to update the older block key with new key generator
- blockExecutionInfo.setKeyStructureInfo(queryProperties.keyStructureInfo);
- // setting the size of fixed key column (dictionary column)
- blockExecutionInfo.setFixedLengthKeySize(getKeySize(updatedQueryDimension, segmentProperties));
- Set<Integer> dictionaryColumnBlockIndex = new HashSet<Integer>();
- List<Integer> noDictionaryColumnBlockIndex = new ArrayList<Integer>();
- // get the block index to be read from file for query dimension
- // for both dictionary columns and no dictionary columns
- QueryUtil.fillQueryDimensionsBlockIndexes(updatedQueryDimension,
- segmentProperties.getDimensionOrdinalToBlockMapping(), dictionaryColumnBlockIndex,
- noDictionaryColumnBlockIndex);
- int[] queryDictionaryColumnBlockIndexes = ArrayUtils.toPrimitive(
- dictionaryColumnBlockIndex.toArray(new Integer[dictionaryColumnBlockIndex.size()]));
- // need to sort the dictionary column as for all dimension
- // column key will be filled based on key order
- Arrays.sort(queryDictionaryColumnBlockIndexes);
- blockExecutionInfo.setDictionaryColumnBlockIndex(queryDictionaryColumnBlockIndexes);
- // setting the no dictionary column block indexes
- blockExecutionInfo.setNoDictionaryBlockIndexes(ArrayUtils.toPrimitive(
- noDictionaryColumnBlockIndex.toArray(new Integer[noDictionaryColumnBlockIndex.size()])));
- // setting column id to dictionary mapping
- blockExecutionInfo.setColumnIdToDcitionaryMapping(queryProperties.columnToDictionayMapping);
- // setting each column value size
- blockExecutionInfo.setEachColumnValueSize(segmentProperties.getEachDimColumnValueSize());
- blockExecutionInfo.setComplexColumnParentBlockIndexes(
- getComplexDimensionParentBlockIndexes(updatedQueryDimension));
- blockExecutionInfo.setVectorBatchCollector(queryModel.isVectorReader());
- try {
- // to set column group and its key structure info which will be used
- // to
- // for getting the column group column data in case of final row
- // and in case of dimension aggregation
- blockExecutionInfo.setColumnGroupToKeyStructureInfo(
- QueryUtil.getColumnGroupKeyStructureInfo(updatedQueryDimension, segmentProperties));
- } catch (KeyGenException e) {
- throw new QueryExecutionException(e);
- }
- return blockExecutionInfo;
- }
-
- /**
- * This method will be used to get fixed key length size this will be used
- * to create a row from column chunk
- *
- * @param queryDimension query dimension
- * @param blockMetadataInfo block metadata info
- * @return key size
- */
- private int getKeySize(List<QueryDimension> queryDimension, SegmentProperties blockMetadataInfo) {
- List<Integer> fixedLengthDimensionOrdinal =
- new ArrayList<Integer>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- int counter = 0;
- while (counter < queryDimension.size()) {
- if (queryDimension.get(counter).getDimension().numberOfChild() > 0) {
- counter += queryDimension.get(counter).getDimension().numberOfChild();
- continue;
- } else if (!CarbonUtil.hasEncoding(queryDimension.get(counter).getDimension().getEncoder(),
- Encoding.DICTIONARY)) {
- counter++;
- } else {
- fixedLengthDimensionOrdinal.add(queryDimension.get(counter).getDimension().getKeyOrdinal());
- counter++;
- }
- }
- int[] dictioanryColumnOrdinal = ArrayUtils.toPrimitive(
- fixedLengthDimensionOrdinal.toArray(new Integer[fixedLengthDimensionOrdinal.size()]));
- if (dictioanryColumnOrdinal.length > 0) {
- return blockMetadataInfo.getFixedLengthKeySplitter()
- .getKeySizeByBlock(dictioanryColumnOrdinal);
- }
- return 0;
- }
-
- /**
- * Below method will be used to get the aggrgator info for the query
- *
- * @param queryModel query model
- * @param tableBlock table block
- * @return aggregator info
- */
- private AggregatorInfo getAggregatorInfoForBlock(QueryModel queryModel,
- AbstractIndex tableBlock) {
- // getting the aggregate infos which will be used during aggregation
- AggregatorInfo aggregatorInfos = RestructureUtil
- .getAggregatorInfos(queryModel.getQueryMeasures(),
- tableBlock.getSegmentProperties().getMeasures());
- // setting the measure aggregator for all aggregation function selected
- // in query
- aggregatorInfos.setMeasureDataTypes(queryProperties.measureDataTypes);
- return aggregatorInfos;
- }
-
- private int[] getComplexDimensionParentBlockIndexes(List<QueryDimension> queryDimensions) {
- List<Integer> parentBlockIndexList = new ArrayList<Integer>();
- for (QueryDimension queryDimension : queryDimensions) {
- if (CarbonUtil.hasDataType(queryDimension.getDimension().getDataType(),
- new DataType[] { DataType.ARRAY, DataType.STRUCT, DataType.MAP })) {
- parentBlockIndexList.add(queryDimension.getDimension().getOrdinal());
- }
- }
- return ArrayUtils
- .toPrimitive(parentBlockIndexList.toArray(new Integer[parentBlockIndexList.size()]));
- }
-
- /**
- * Below method will be used to finish the execution
- *
- * @throws QueryExecutionException
- */
- @Override public void finish() throws QueryExecutionException {
- CarbonUtil.clearBlockCache(queryProperties.dataBlocks);
- if (null != queryProperties.executorService) {
- queryProperties.executorService.shutdown();
- try {
- queryProperties.executorService.awaitTermination(1, TimeUnit.HOURS);
- } catch (InterruptedException e) {
- throw new QueryExecutionException(e);
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/executor/impl/DetailQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/impl/DetailQueryExecutor.java b/core/src/main/java/org/apache/carbondata/scan/executor/impl/DetailQueryExecutor.java
deleted file mode 100644
index 92e7fc8..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/executor/impl/DetailQueryExecutor.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.executor.impl;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
-import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.scan.model.QueryModel;
-import org.apache.carbondata.scan.result.BatchResult;
-import org.apache.carbondata.scan.result.iterator.DetailQueryResultIterator;
-
-/**
- * Below class will be used to execute the detail query
- * For executing the detail query it will pass all the block execution
- * info to detail query result iterator and iterator will be returned
- */
-public class DetailQueryExecutor extends AbstractQueryExecutor<BatchResult> {
-
- @Override
- public CarbonIterator<BatchResult> execute(QueryModel queryModel)
- throws QueryExecutionException, IOException {
- List<BlockExecutionInfo> blockExecutionInfoList = getBlockExecutionInfos(queryModel);
- return new DetailQueryResultIterator(
- blockExecutionInfoList,
- queryModel,
- queryProperties.executorService
- );
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/executor/impl/QueryExecutorProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/impl/QueryExecutorProperties.java b/core/src/main/java/org/apache/carbondata/scan/executor/impl/QueryExecutorProperties.java
deleted file mode 100644
index 5683f62..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/executor/impl/QueryExecutorProperties.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.executor.impl;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
-
-import org.apache.carbondata.core.cache.dictionary.Dictionary;
-import org.apache.carbondata.core.datastore.block.AbstractIndex;
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-import org.apache.carbondata.core.stats.QueryStatisticsRecorder;
-import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
-
-/**
- * Holds all the properties required for query execution
- */
-public class QueryExecutorProperties {
-
- /**
- * holds the information required for updating the order block
- * dictionary key
- */
- public KeyStructureInfo keyStructureInfo;
-
- /**
- * this will hold the information about the dictionary dimension
- * which to
- */
- public Map<String, Dictionary> columnToDictionayMapping;
-
- /**
- * Measure datatypes
- */
- public DataType[] measureDataTypes;
- /**
- * all the complex dimension which is on filter
- */
- public Set<CarbonDimension> complexFilterDimension;
-
- public Set<CarbonMeasure> filterMeasures;
- /**
- * to record the query execution details phase wise
- */
- public QueryStatisticsRecorder queryStatisticsRecorder;
- /**
- * executor service to execute the query
- */
- public ExecutorService executorService;
- /**
- * list of blocks in which query will be executed
- */
- protected List<AbstractIndex> dataBlocks;
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/executor/impl/VectorDetailQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/impl/VectorDetailQueryExecutor.java b/core/src/main/java/org/apache/carbondata/scan/executor/impl/VectorDetailQueryExecutor.java
deleted file mode 100644
index 1f1dd7d..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/executor/impl/VectorDetailQueryExecutor.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.executor.impl;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
-import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.scan.model.QueryModel;
-import org.apache.carbondata.scan.result.iterator.VectorDetailQueryResultIterator;
-
-/**
- * Below class will be used to execute the detail query and returns columnar vectors.
- */
-public class VectorDetailQueryExecutor extends AbstractQueryExecutor<Object> {
-
- @Override
- public CarbonIterator<Object> execute(QueryModel queryModel)
- throws QueryExecutionException, IOException {
- List<BlockExecutionInfo> blockExecutionInfoList = getBlockExecutionInfos(queryModel);
- return new VectorDetailQueryResultIterator(
- blockExecutionInfoList,
- queryModel,
- queryProperties.executorService
- );
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/executor/infos/AggregatorInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/infos/AggregatorInfo.java b/core/src/main/java/org/apache/carbondata/scan/executor/infos/AggregatorInfo.java
deleted file mode 100644
index 244bd2b..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/executor/infos/AggregatorInfo.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.executor.infos;
-
-import org.apache.carbondata.core.metadata.DataType;
-
-/**
- * Info class which store all the details
- * which is required during aggregation
- */
-public class AggregatorInfo {
-
- /**
- * selected query measure ordinal
- * which will be used to read the measures chunk data
- * this will be storing the index of the measure in measures chunk
- */
- private int[] measureOrdinals;
-
- /**
- * This parameter will be used to
- * check whether particular measure is present
- * in the table block, if not then its default value will be used
- */
- private boolean[] measureExists;
-
- /**
- * this default value will be used to when some measure is not present
- * in the table block, in case of restructuring of the table if user is adding any
- * measure then in older block that measure wont be present so for measure default value
- * will be used to aggregate in the older table block query execution
- */
- private Object[] defaultValues;
-
- /**
- * Datatype of each measure;
- */
- private DataType[] measureDataTypes;
-
- /**
- * @return the measureOrdinal
- */
- public int[] getMeasureOrdinals() {
- return measureOrdinals;
- }
-
- /**
- * @param measureOrdinal the measureOrdinal to set
- */
- public void setMeasureOrdinals(int[] measureOrdinal) {
- this.measureOrdinals = measureOrdinal;
- }
-
- /**
- * @return the measureExists
- */
- public boolean[] getMeasureExists() {
- return measureExists;
- }
-
- /**
- * @param measureExists the measureExists to set
- */
- public void setMeasureExists(boolean[] measureExists) {
- this.measureExists = measureExists;
- }
-
- /**
- * @return the defaultValues
- */
- public Object[] getDefaultValues() {
- return defaultValues;
- }
-
- /**
- * @param defaultValues the defaultValues to set
- */
- public void setDefaultValues(Object[] defaultValues) {
- this.defaultValues = defaultValues;
- }
-
- public DataType[] getMeasureDataTypes() {
- return measureDataTypes;
- }
-
- public void setMeasureDataTypes(DataType[] measureDataTypes) {
- this.measureDataTypes = measureDataTypes;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/executor/infos/BlockExecutionInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/infos/BlockExecutionInfo.java b/core/src/main/java/org/apache/carbondata/scan/executor/infos/BlockExecutionInfo.java
deleted file mode 100644
index 59b8acf..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/executor/infos/BlockExecutionInfo.java
+++ /dev/null
@@ -1,604 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.executor.infos;
-
-import java.util.Map;
-
-import org.apache.carbondata.core.cache.dictionary.Dictionary;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.datastore.DataRefNode;
-import org.apache.carbondata.core.datastore.IndexKey;
-import org.apache.carbondata.core.datastore.block.AbstractIndex;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
-import org.apache.carbondata.scan.filter.GenericQueryType;
-import org.apache.carbondata.scan.filter.executer.FilterExecuter;
-import org.apache.carbondata.scan.model.QueryDimension;
-import org.apache.carbondata.scan.model.QueryMeasure;
-
-/**
- * Below class will have all the properties which needed during query execution
- * for one block
- */
-public class BlockExecutionInfo {
-
- /**
- * block on which query will be executed
- */
- private AbstractIndex blockIndex;
-
- /**
- * each segment key size can be different and in that case we need to update
- * the fixed key with latest segment key generator. so this property will
- * tell whether this is required or not if key size is same then it is not
- * required
- */
- private boolean isFixedKeyUpdateRequired;
-
- /**
- * below to store all the information required for aggregation during query
- * execution
- */
- private AggregatorInfo aggregatorInfo;
-
- /**
- * this will be used to get the first tentative block from which query
- * execution start, this will be useful in case of filter query to get the
- * start block based on filter values
- */
- private IndexKey startKey;
-
- /**
- * this will be used to get the last tentative block till which scanning
- * will be done, this will be useful in case of filter query to get the last
- * block based on filter values
- */
- private IndexKey endKey;
-
- private String blockId;
-
- /**
- * masked byte for block which will be used to unpack the fixed length key,
- * this will be used for updating the older block key with new block key
- * generator
- */
- private int[] maskedByteForBlock;
-
- /**
- * total number of dimension in block
- */
- private int totalNumberDimensionBlock;
-
- /**
- * total number of measure in block
- */
- private int totalNumberOfMeasureBlock;
-
- /**
- * will be used to read the dimension block from file
- */
- private int[][] allSelectedDimensionBlocksIndexes;
-
- /**
- * will be used to read the measure block from file
- */
- private int[][] allSelectedMeasureBlocksIndexes;
-
- /**
- * this will be used to update the older block fixed length keys with the
- * new block fixed length key
- */
- private KeyStructureInfo keyStructureInfo;
-
- /**
- * first block from which query execution will start
- */
- private DataRefNode firstDataBlock;
-
- /**
- * number of block to be scanned in the query
- */
- private long numberOfBlockToScan;
-
- /**
- * key size of the fixed length dimension column
- */
- private int fixedLengthKeySize;
-
- /**
- * dictionary column block indexes based on query
- */
- private int[] dictionaryColumnBlockIndex;
- /**
- * no dictionary column block indexes in based on the query order
- */
- private int[] noDictionaryBlockIndexes;
-
- /**
- * key generator used for generating the table block fixed length key
- */
- private KeyGenerator blockKeyGenerator;
-
- /**
- * each column value size
- */
- private int[] eachColumnValueSize;
-
- /**
- * column group block index in file to key structure info mapping
- */
- private Map<Integer, KeyStructureInfo> columnGroupToKeyStructureInfo;
-
- /**
- * mapping of dictionary dimension to its dictionary mapping which will be
- * used to get the actual data from dictionary for aggregation, sorting
- */
- private Map<String, Dictionary> columnIdToDcitionaryMapping;
-
- /**
- * filter tree to execute the filter
- */
- private FilterExecuter filterExecuterTree;
-
- /**
- * whether it needs only raw byte records with out aggregation.
- */
- private boolean isRawRecordDetailQuery;
-
- /**
- * start index of blocklets
- */
- private int startBlockletIndex;
-
- /**
- * number of blocklet to be scanned
- */
- private int numberOfBlockletToScan;
-
- /**
- * complexParentIndexToQueryMap
- */
- private Map<Integer, GenericQueryType> complexParentIndexToQueryMap;
-
- /**
- * complex dimension parent block indexes;
- */
- private int[] complexColumnParentBlockIndexes;
-
- /**
- * @return the tableBlock
- */
- public AbstractIndex getDataBlock() {
- return blockIndex;
- }
-
- /**
- * list of dimension selected for in query
- */
- private QueryDimension[] queryDimensions;
-
- /**
- * list of measure selected in query
- */
- private QueryMeasure[] queryMeasures;
-
- /**
- * whether it needs to read data in vector/columnar format.
- */
- private boolean vectorBatchCollector;
-
- /**
- * absolute table identifier
- */
- private AbsoluteTableIdentifier absoluteTableIdentifier;
-
- public AbsoluteTableIdentifier getAbsoluteTableIdentifier() {
- return absoluteTableIdentifier;
- }
-
- public void setAbsoluteTableIdentifier(AbsoluteTableIdentifier absoluteTableIdentifier) {
- this.absoluteTableIdentifier = absoluteTableIdentifier;
- }
-
- /**
- * @param blockIndex the tableBlock to set
- */
- public void setDataBlock(AbstractIndex blockIndex) {
- this.blockIndex = blockIndex;
- }
-
- /**
- * @return the isFixedKeyUpdateRequired
- */
- public boolean isFixedKeyUpdateRequired() {
- return isFixedKeyUpdateRequired;
- }
-
- /**
- * @param isFixedKeyUpdateRequired the isFixedKeyUpdateRequired to set
- */
- public void setFixedKeyUpdateRequired(boolean isFixedKeyUpdateRequired) {
- this.isFixedKeyUpdateRequired = isFixedKeyUpdateRequired;
- }
-
- /**
- * @return the aggregatorInfos
- */
- public AggregatorInfo getAggregatorInfo() {
- return aggregatorInfo;
- }
-
- /**
- * @param aggregatorInfo the aggregatorInfos to set
- */
- public void setAggregatorInfo(AggregatorInfo aggregatorInfo) {
- this.aggregatorInfo = aggregatorInfo;
- }
-
- /**
- * @return the startKey
- */
- public IndexKey getStartKey() {
- return startKey;
- }
-
- /**
- * @param startKey the startKey to set
- */
- public void setStartKey(IndexKey startKey) {
- this.startKey = startKey;
- }
-
- /**
- * @return the endKey
- */
- public IndexKey getEndKey() {
- return endKey;
- }
-
- /**
- * @param endKey the endKey to set
- */
- public void setEndKey(IndexKey endKey) {
- this.endKey = endKey;
- }
-
- /**
- * @return the maskedByteForBlock
- */
- public int[] getMaskedByteForBlock() {
- return maskedByteForBlock;
- }
-
-
-
- /**
- * @param maskedByteForBlock the maskedByteForBlock to set
- */
- public void setMaskedByteForBlock(int[] maskedByteForBlock) {
- this.maskedByteForBlock = maskedByteForBlock;
- }
-
- /**
- * @return the totalNumberDimensionBlock
- */
- public int getTotalNumberDimensionBlock() {
- return totalNumberDimensionBlock;
- }
-
- /**
- * @param totalNumberDimensionBlock the totalNumberDimensionBlock to set
- */
- public void setTotalNumberDimensionBlock(int totalNumberDimensionBlock) {
- this.totalNumberDimensionBlock = totalNumberDimensionBlock;
- }
-
- /**
- * @return the totalNumberOfMeasureBlock
- */
- public int getTotalNumberOfMeasureBlock() {
- return totalNumberOfMeasureBlock;
- }
-
- /**
- * @param totalNumberOfMeasureBlock the totalNumberOfMeasureBlock to set
- */
- public void setTotalNumberOfMeasureBlock(int totalNumberOfMeasureBlock) {
- this.totalNumberOfMeasureBlock = totalNumberOfMeasureBlock;
- }
-
- /**
- * @return the allSelectedDimensionBlocksIndexes
- */
- public int[][] getAllSelectedDimensionBlocksIndexes() {
- return allSelectedDimensionBlocksIndexes;
- }
-
- /**
- * @param allSelectedDimensionBlocksIndexes the allSelectedDimensionBlocksIndexes to set
- */
- public void setAllSelectedDimensionBlocksIndexes(int[][] allSelectedDimensionBlocksIndexes) {
- this.allSelectedDimensionBlocksIndexes = allSelectedDimensionBlocksIndexes;
- }
-
- /**
- * @return the allSelectedMeasureBlocksIndexes
- */
- public int[][] getAllSelectedMeasureBlocksIndexes() {
- return allSelectedMeasureBlocksIndexes;
- }
-
- /**
- * @param allSelectedMeasureBlocksIndexes the allSelectedMeasureBlocksIndexes to set
- */
- public void setAllSelectedMeasureBlocksIndexes(int[][] allSelectedMeasureBlocksIndexes) {
- this.allSelectedMeasureBlocksIndexes = allSelectedMeasureBlocksIndexes;
- }
-
- /**
- * @return the restructureInfos
- */
- public KeyStructureInfo getKeyStructureInfo() {
- return keyStructureInfo;
- }
-
- /**
- * @param keyStructureInfo the restructureInfos to set
- */
- public void setKeyStructureInfo(KeyStructureInfo keyStructureInfo) {
- this.keyStructureInfo = keyStructureInfo;
- }
-
- /**
- * @return the firstDataBlock
- */
- public DataRefNode getFirstDataBlock() {
- return firstDataBlock;
- }
-
- /**
- * @param firstDataBlock the firstDataBlock to set
- */
- public void setFirstDataBlock(DataRefNode firstDataBlock) {
- this.firstDataBlock = firstDataBlock;
- }
-
- /**
- * @return the numberOfBlockToScan
- */
- public long getNumberOfBlockToScan() {
- return numberOfBlockToScan;
- }
-
- /**
- * @param numberOfBlockToScan the numberOfBlockToScan to set
- */
- public void setNumberOfBlockToScan(long numberOfBlockToScan) {
- this.numberOfBlockToScan = numberOfBlockToScan;
- }
-
- /**
- * @return the fixedLengthKeySize
- */
- public int getFixedLengthKeySize() {
- return fixedLengthKeySize;
- }
-
- /**
- * @param fixedLengthKeySize the fixedLengthKeySize to set
- */
- public void setFixedLengthKeySize(int fixedLengthKeySize) {
- this.fixedLengthKeySize = fixedLengthKeySize;
- }
-
- /**
- * @return the filterEvaluatorTree
- */
- public FilterExecuter getFilterExecuterTree() {
- return filterExecuterTree;
- }
-
- /**
- * @param filterExecuterTree the filterEvaluatorTree to set
- */
- public void setFilterExecuterTree(FilterExecuter filterExecuterTree) {
- this.filterExecuterTree = filterExecuterTree;
- }
-
- /**
- * @return the tableBlockKeyGenerator
- */
- public KeyGenerator getBlockKeyGenerator() {
- return blockKeyGenerator;
- }
-
- /**
- * @param tableBlockKeyGenerator the tableBlockKeyGenerator to set
- */
- public void setBlockKeyGenerator(KeyGenerator tableBlockKeyGenerator) {
- this.blockKeyGenerator = tableBlockKeyGenerator;
- }
-
- /**
- * @return the eachColumnValueSize
- */
- public int[] getEachColumnValueSize() {
- return eachColumnValueSize;
- }
-
- /**
- * @param eachColumnValueSize the eachColumnValueSize to set
- */
- public void setEachColumnValueSize(int[] eachColumnValueSize) {
- this.eachColumnValueSize = eachColumnValueSize;
- }
-
- /**
- * @return the dictionaryColumnBlockIndex
- */
- public int[] getDictionaryColumnBlockIndex() {
- return dictionaryColumnBlockIndex;
- }
-
- /**
- * @param dictionaryColumnBlockIndex the dictionaryColumnBlockIndex to set
- */
- public void setDictionaryColumnBlockIndex(int[] dictionaryColumnBlockIndex) {
- this.dictionaryColumnBlockIndex = dictionaryColumnBlockIndex;
- }
-
- /**
- * @return the noDictionaryBlockIndexes
- */
- public int[] getNoDictionaryBlockIndexes() {
- return noDictionaryBlockIndexes;
- }
-
- /**
- * @param noDictionaryBlockIndexes the noDictionaryBlockIndexes to set
- */
- public void setNoDictionaryBlockIndexes(int[] noDictionaryBlockIndexes) {
- this.noDictionaryBlockIndexes = noDictionaryBlockIndexes;
- }
-
- /**
- * @return the columnGroupToKeyStructureInfo
- */
- public Map<Integer, KeyStructureInfo> getColumnGroupToKeyStructureInfo() {
- return columnGroupToKeyStructureInfo;
- }
-
- /**
- * @param columnGroupToKeyStructureInfo the columnGroupToKeyStructureInfo to set
- */
- public void setColumnGroupToKeyStructureInfo(
- Map<Integer, KeyStructureInfo> columnGroupToKeyStructureInfo) {
- this.columnGroupToKeyStructureInfo = columnGroupToKeyStructureInfo;
- }
-
- /**
- * @param columnIdToDcitionaryMapping the columnIdToDcitionaryMapping to set
- */
- public void setColumnIdToDcitionaryMapping(Map<String, Dictionary> columnIdToDcitionaryMapping) {
- this.columnIdToDcitionaryMapping = columnIdToDcitionaryMapping;
- }
-
- public boolean isRawRecordDetailQuery() {
- return isRawRecordDetailQuery;
- }
-
- public void setRawRecordDetailQuery(boolean rawRecordDetailQuery) {
- isRawRecordDetailQuery = rawRecordDetailQuery;
- }
-
- /**
- * @return the complexParentIndexToQueryMap
- */
- public Map<Integer, GenericQueryType> getComlexDimensionInfoMap() {
- return complexParentIndexToQueryMap;
- }
-
- /**
- * @param complexDimensionInfoMap the complexParentIndexToQueryMap to set
- */
- public void setComplexDimensionInfoMap(Map<Integer, GenericQueryType> complexDimensionInfoMap) {
- this.complexParentIndexToQueryMap = complexDimensionInfoMap;
- }
-
- /**
- * @return the complexColumnParentBlockIndexes
- */
- public int[] getComplexColumnParentBlockIndexes() {
- return complexColumnParentBlockIndexes;
- }
-
- /**
- * @param complexColumnParentBlockIndexes the complexColumnParentBlockIndexes to set
- */
- public void setComplexColumnParentBlockIndexes(int[] complexColumnParentBlockIndexes) {
- this.complexColumnParentBlockIndexes = complexColumnParentBlockIndexes;
- }
-
- public QueryDimension[] getQueryDimensions() {
- return queryDimensions;
- }
-
- public void setQueryDimensions(QueryDimension[] queryDimensions) {
- this.queryDimensions = queryDimensions;
- }
-
- public QueryMeasure[] getQueryMeasures() {
- return queryMeasures;
- }
-
- public void setQueryMeasures(QueryMeasure[] queryMeasures) {
- this.queryMeasures = queryMeasures;
- }
-
- /**
- * The method to set the number of blocklets to be scanned
- *
- * @param numberOfBlockletToScan
- */
- public void setNumberOfBlockletToScan(int numberOfBlockletToScan) {
- this.numberOfBlockletToScan = numberOfBlockletToScan;
- }
-
- /**
- * get the no of blocklet to be scanned
- *
- * @return
- */
- public int getNumberOfBlockletToScan() {
- return numberOfBlockletToScan;
- }
-
- /**
- * returns the blocklet index to be scanned
- *
- * @return
- */
- public int getStartBlockletIndex() {
- return startBlockletIndex;
- }
-
- /**
- * set the blocklet index to be scanned
- *
- * @param startBlockletIndex
- */
- public void setStartBlockletIndex(int startBlockletIndex) {
- this.startBlockletIndex = startBlockletIndex;
- }
-
- public boolean isVectorBatchCollector() {
- return vectorBatchCollector;
- }
-
- public void setVectorBatchCollector(boolean vectorBatchCollector) {
- this.vectorBatchCollector = vectorBatchCollector;
- }
-
- public String getBlockId() {
- return blockId;
- }
-
- public void setBlockId(String blockId) {
- this.blockId = blockId;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/executor/infos/KeyStructureInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/infos/KeyStructureInfo.java b/core/src/main/java/org/apache/carbondata/scan/executor/infos/KeyStructureInfo.java
deleted file mode 100644
index d742e02..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/executor/infos/KeyStructureInfo.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.executor.infos;
-
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
-
-/**
- * Below class will store the structure of the key
- * used during query execution
- */
-public class KeyStructureInfo {
-
- /**
- * it's actually a latest key generator
- * last table block as this key generator will be used to
- * to update the mdkey of the older slice with the new slice
- */
- private KeyGenerator keyGenerator;
-
- /**
- * mask bytes ranges for the query
- */
- private int[] maskByteRanges;
-
- /**
- * max key for query execution
- */
- private byte[] maxKey;
-
- /**
- * dimension ordinals inside the column group
- */
- private int[] mdkeyQueryDimensionOrdinal;
-
- /**
- * @return the keyGenerator
- */
- public KeyGenerator getKeyGenerator() {
- return keyGenerator;
- }
-
- /**
- * @param keyGenerator the keyGenerator to set
- */
- public void setKeyGenerator(KeyGenerator keyGenerator) {
- this.keyGenerator = keyGenerator;
- }
-
- /**
- * @return the maskByteRanges
- */
- public int[] getMaskByteRanges() {
- return maskByteRanges;
- }
-
- /**
- * @param maskByteRanges the maskByteRanges to set
- */
- public void setMaskByteRanges(int[] maskByteRanges) {
- this.maskByteRanges = maskByteRanges;
- }
-
- /**
- * @return the maxKey
- */
- public byte[] getMaxKey() {
- return maxKey;
- }
-
- /**
- * @param maxKey the maxKey to set
- */
- public void setMaxKey(byte[] maxKey) {
- this.maxKey = maxKey;
- }
-
- public int[] getMdkeyQueryDimensionOrdinal() {
- return mdkeyQueryDimensionOrdinal;
- }
-
- public void setMdkeyQueryDimensionOrdinal(int[] mdkeyQueryDimensionOrdinal) {
- this.mdkeyQueryDimensionOrdinal = mdkeyQueryDimensionOrdinal;
- }
-}
[03/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scan/filter/FilterUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/filter/FilterUtilTest.java b/core/src/test/java/org/apache/carbondata/scan/filter/FilterUtilTest.java
deleted file mode 100644
index cdd530d..0000000
--- a/core/src/test/java/org/apache/carbondata/scan/filter/FilterUtilTest.java
+++ /dev/null
@@ -1,392 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.filter;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.UUID;
-
-import org.apache.carbondata.core.cache.dictionary.AbstractDictionaryCacheTest;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.datastore.IndexKey;
-import org.apache.carbondata.core.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.core.metadata.Encoding;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.LiteralExpression;
-import org.apache.carbondata.scan.expression.conditional.ListExpression;
-import org.apache.carbondata.scan.filter.intf.RowImpl;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.junit.Before;
-import org.junit.Test;
-
-import static junit.framework.TestCase.assertFalse;
-import static junit.framework.TestCase.assertTrue;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-
-public class FilterUtilTest extends AbstractDictionaryCacheTest {
-
- private ColumnSchema columnSchema;
-
- @Before public void setUp() throws Exception {
- init();
- this.databaseName = props.getProperty("database", "testSchema");
- this.tableName = props.getProperty("tableName", "carbon");
- this.carbonStorePath = props.getProperty("storePath", "carbonStore");
- carbonTableIdentifier =
- new CarbonTableIdentifier(databaseName, tableName, UUID.randomUUID().toString());
- this.carbonStorePath = props.getProperty("storePath", "carbonStore");
- columnSchema = new ColumnSchema();
- columnSchema.setColumnar(true);
- columnSchema.setColumnName("IMEI");
- columnSchema.setColumnUniqueId(UUID.randomUUID().toString());
- columnSchema.setDataType(DataType.STRING);
- columnSchema.setDimensionColumn(true);
- }
-
- @Test public void testCheckIfLeftExpressionRequireEvaluation() {
- List<Expression> children = new ArrayList<>();
- ListExpression expression = new ListExpression(children);
- boolean result = FilterUtil.checkIfLeftExpressionRequireEvaluation(expression);
- assertTrue(result);
- }
-
- @Test
- public void testCheckIfLeftExpressionRequireEvaluationWithExpressionNotInstanceOfColumnExpression() {
- ColumnExpression expression = new ColumnExpression("test", DataType.STRING);
- boolean result = FilterUtil.checkIfLeftExpressionRequireEvaluation(expression);
- assertFalse(result);
- }
-
- @Test public void testNanSafeEqualsDoublesWithUnEqualValues() {
- Double d1 = new Double(60.67);
- Double d2 = new Double(60.69);
- boolean result = FilterUtil.nanSafeEqualsDoubles(d1, d2);
- assertFalse(result);
- }
-
- @Test public void testNanSafeEqualsDoublesWithEqualValues() {
- Double d1 = new Double(60.67);
- Double d2 = new Double(60.67);
- boolean result = FilterUtil.nanSafeEqualsDoubles(d1, d2);
- assertTrue(result);
- }
-
- @Test public void testCompareFilterKeyBasedOnDataTypeForShortValue() {
- String dictionaryVal = "1";
- String memberVal = "1";
- int actualResult =
- FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataType.SHORT);
- int expectedResult = 0;
- assertEquals(expectedResult, actualResult);
- }
-
- @Test public void testCompareFilterKeyBasedOnDataTypeForIntValue() {
- String dictionaryVal = "1000";
- String memberVal = "1001";
- int actualResult =
- FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataType.INT);
- int expectedResult = -1;
- assertEquals(expectedResult, actualResult);
- }
-
- @Test public void testCompareFilterKeyBasedOnDataTypeForDoubleValue() {
- String dictionaryVal = "1.90";
- String memberVal = "1.89";
- int actualResult =
- FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataType.DOUBLE);
- int expectedResult = 1;
- assertEquals(expectedResult, actualResult);
- }
-
- @Test public void testCompareFilterKeyBasedOnDataTypeForLongValue() {
- String dictionaryVal = "111111111111111";
- String memberVal = "1111111111111111";
- int actualResult =
- FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataType.LONG);
- int expectedResult = -1;
- assertEquals(expectedResult, actualResult);
- }
-
- @Test public void testCompareFilterKeyBasedOnDataTypeForBooleanValue() {
- String dictionaryVal = "true";
- String memberVal = "false";
- int actualResult =
- FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataType.BOOLEAN);
- int expectedResult = 1;
- assertEquals(expectedResult, actualResult);
- }
-
- @Test public void testCompareFilterKeyBasedOnDataTypeForDecimalValue() {
- String dictionaryVal = "1111111";
- String memberVal = "1111";
- int actualResult =
- FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataType.DECIMAL);
- int expectedResult = 1;
- assertEquals(expectedResult, actualResult);
- }
-
- @Test public void testCompareFilterKeyBasedOnDataTypeForDefaultValue() {
- String dictionaryVal = "11.78";
- String memberVal = "1111.90";
- int actualResult =
- FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataType.FLOAT);
- int expectedResult = -1;
- assertEquals(expectedResult, actualResult);
- }
-
- @Test public void testCompareFilterKeyBasedOnDataTypeForTimestamp() {
- String dictionaryVal = "2008-01-01 00:00:01";
- String memberVal = "2008-01-01 00:00:01";
- int actualValue =
- FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataType.TIMESTAMP);
- int expectedValue = 0;
- assertEquals(expectedValue, actualValue);
- }
-
- @Test public void testCompareFilterKeyBasedOnDataTypeForException() throws Exception {
- String dictionaryVal = "test";
- String memberVal = "1";
- int actualValue =
- FilterUtil.compareFilterKeyBasedOnDataType(dictionaryVal, memberVal, DataType.INT);
- int expectedValue = -1;
- assertEquals(expectedValue, actualValue);
- }
-
- @Test public void testCreateIndexKeyFromResolvedFilterVal() throws Exception {
- long[] startOrEndKey = new long[] { 0, 10 };
- byte[] startOrEndKeyForNoDictDimension = { 1, 2 };
- int[] keys = new int[] { 1, 2 };
- MultiDimKeyVarLengthGenerator multiDimKeyVarLengthGenerator =
- new MultiDimKeyVarLengthGenerator(keys);
- assertTrue(FilterUtil
- .createIndexKeyFromResolvedFilterVal(startOrEndKey, multiDimKeyVarLengthGenerator,
- startOrEndKeyForNoDictDimension) != null);
-
- }
-
- @Test public void testCheckIfExpressionContainsColumn() {
- String columnName = "IMEI";
- Expression expression = new ColumnExpression(columnName, DataType.STRING);
- boolean result = FilterUtil.checkIfExpressionContainsColumn(expression);
- assertTrue(result);
- }
-
- @Test
- public void testCheckIfExpressionContainsColumnWithExpressionNotInstanceOfColumnExpression() {
- String columnName = "IMEI";
- Expression expression = new LiteralExpression(columnName, DataType.STRING);
- boolean result = FilterUtil.checkIfExpressionContainsColumn(expression);
- assertFalse(result);
- }
-
- @Test public void testIsExpressionNeedsToResolved() {
- boolean isIncludeFilter = true;
- Object obj = "test";
- LiteralExpression literalExpression = new LiteralExpression(obj, DataType.STRING);
- boolean result = FilterUtil.isExpressionNeedsToResolved(literalExpression, isIncludeFilter);
- assertFalse(result);
- }
-
- @Test public void testIsExpressionNeedsToResolvedWithDataTypeNullAndIsIncludeFilterFalse() {
- boolean isIncludeFilter = false;
- Object obj = "test";
- LiteralExpression literalExpression = new LiteralExpression(obj, DataType.NULL);
- boolean result = FilterUtil.isExpressionNeedsToResolved(literalExpression, isIncludeFilter);
- assertTrue(result);
- }
-
- @Test public void testGetMaskKey() {
- int surrogate = 1;
- int[] keys = new int[] { 1, 2 };
- MultiDimKeyVarLengthGenerator multiDimKeyVarLengthGenerator =
- new MultiDimKeyVarLengthGenerator(keys);
- int ordinal = 1;
- int keyOrdinal = 1;
- int columnGroupOrdinal = 1;
- int complexTypeOrdinal = 1;
- ColumnSchema columnSchema = new ColumnSchema();
- columnSchema.setColumnar(true);
- columnSchema.setColumnName("IMEI");
- columnSchema.setColumnUniqueId(UUID.randomUUID().toString());
- columnSchema.setDataType(DataType.STRING);
- columnSchema.setDimensionColumn(true);
- CarbonDimension carbonDimension =
- new CarbonDimension(columnSchema, ordinal, keyOrdinal, columnGroupOrdinal,
- complexTypeOrdinal);
- byte[] expectedResult = new byte[] { 1 };
- byte[] actualResult =
- FilterUtil.getMaskKey(surrogate, carbonDimension, multiDimKeyVarLengthGenerator);
- assertArrayEquals(expectedResult, actualResult);
- }
-
- @Test public void testGetFilterListForAllMembersRS() throws Exception {
- Expression expression = new ColumnExpression("IMEI", DataType.STRING);
- ColumnExpression columnExpression = new ColumnExpression("IMEI", DataType.STRING);
- String defaultValues = "test";
- int defaultSurrogate = 1;
- boolean isIncludeFilter = true;
- int ordinal = 1;
- ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setColumnar(true);
- dimColumn.setColumnName("IMEI");
- dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
- dimColumn.setDataType(DataType.STRING);
- dimColumn.setDimensionColumn(true);
- final CarbonColumn carbonColumn = new CarbonColumn(dimColumn, ordinal, -1);
- new MockUp<ColumnExpression>() {
- @Mock public CarbonColumn getCarbonColumn() {
- return carbonColumn;
- }
- };
-
- new MockUp<RowImpl>() {
- @Mock public Object getVal(int index) {
- return "test";
- }
- };
- assertTrue(FilterUtil
- .getFilterListForAllMembersRS(expression, columnExpression, defaultValues, defaultSurrogate,
- isIncludeFilter) instanceof DimColumnFilterInfo);
- }
-
- @Test public void testGetFilterListForAllMembersRSWithDefaultValuesEqualsToNull()
- throws Exception {
- Expression expression = new ColumnExpression("IMEI", DataType.STRING);
- ColumnExpression columnExpression = new ColumnExpression("IMEI", DataType.STRING);
- String defaultValues = CarbonCommonConstants.MEMBER_DEFAULT_VAL;
- int defaultSurrogate = 1;
- boolean isIncludeFilter = true;
- int ordinal = 1;
- ColumnSchema dimColumn = new ColumnSchema();
- dimColumn.setColumnar(true);
- dimColumn.setColumnName("IMEI");
- dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
- dimColumn.setDataType(DataType.STRING);
- dimColumn.setDimensionColumn(true);
- final CarbonColumn carbonColumn = new CarbonColumn(dimColumn, ordinal, -1);
- new MockUp<ColumnExpression>() {
- @Mock public CarbonColumn getCarbonColumn() {
- return carbonColumn;
- }
- };
-
- new MockUp<RowImpl>() {
- @Mock public Object getVal(int index) {
- return "test";
- }
- };
- assertTrue(FilterUtil
- .getFilterListForAllMembersRS(expression, columnExpression, defaultValues, defaultSurrogate,
- isIncludeFilter) instanceof DimColumnFilterInfo);
- }
-
- @Test public void testgetFilterListForRS() throws Exception {
- Expression expression = new ColumnExpression("IMEI", DataType.STRING);
- ColumnExpression columnExpression = new ColumnExpression("IMEI", DataType.STRING);
- String defaultValues = CarbonCommonConstants.MEMBER_DEFAULT_VAL;
- int defaultSurrogate = 1;
- int ordinal = 1;
- final CarbonColumn carbonColumn = new CarbonColumn(columnSchema, ordinal, -1);
- new MockUp<ColumnExpression>() {
- @Mock public CarbonColumn getCarbonColumn() {
- return carbonColumn;
- }
- };
-
- new MockUp<RowImpl>() {
- @Mock public Object getVal(int index) {
- return "test";
- }
- };
- assertTrue(FilterUtil.getFilterListForRS(expression, columnExpression, defaultValues,
- defaultSurrogate) instanceof DimColumnFilterInfo);
- }
-
- @Test public void testCheckIfDataTypeNotTimeStamp() {
- Expression expression = new ColumnExpression("test", DataType.STRING);
- boolean result = FilterUtil.checkIfDataTypeNotTimeStamp(expression);
- assertFalse(result);
- }
-
- @Test public void testPrepareDefaultEndIndexKey() throws Exception {
- List<ColumnSchema> columnsInTable = new ArrayList<>();
- columnsInTable.add(columnSchema);
- int[] columnCardinality = new int[] { 1, 2 };
- new MockUp<ColumnSchema>() {
- @Mock public List<Encoding> getEncodingList() {
- List<Encoding> encodingList = new ArrayList<>();
- encodingList.add(Encoding.DICTIONARY);
- return encodingList;
- }
- };
- SegmentProperties segmentProperties = new SegmentProperties(columnsInTable, columnCardinality);
- assertTrue(FilterUtil.prepareDefaultEndIndexKey(segmentProperties) instanceof IndexKey);
- }
-
- @Test public void testCheckIfRightExpressionRequireEvaluation() {
- Expression expression = new ColumnExpression("test", DataType.STRING);
- boolean result = FilterUtil.checkIfRightExpressionRequireEvaluation(expression);
- assertTrue(result);
- }
-
- @Test
- public void testCheckIfRightExpressionRequireEvaluationWithExpressionIsInstanceOfLiteralExpression() {
- Expression expression = new LiteralExpression("test", DataType.STRING);
- boolean result = FilterUtil.checkIfRightExpressionRequireEvaluation(expression);
- assertFalse(result);
- }
-
- @Test public void testGetNoDictionaryValKeyMemberForFilter() {
- boolean isIncludeFilter = true;
- AbsoluteTableIdentifier absoluteTableIdentifier =
- new AbsoluteTableIdentifier(this.carbonStorePath, carbonTableIdentifier);
- ColumnExpression expression = new ColumnExpression("test", DataType.STRING);
- List<String> evaluateResultListFinal = new ArrayList<>();
- evaluateResultListFinal.add("test1");
- evaluateResultListFinal.add("test2");
- assertTrue(FilterUtil.getNoDictionaryValKeyMemberForFilter(evaluateResultListFinal, isIncludeFilter) instanceof DimColumnFilterInfo);
- }
-
- @Test public void testPrepareDefaultStartIndexKey() throws KeyGenException {
- List<ColumnSchema> columnsInTable = new ArrayList<>();
- columnsInTable.add(columnSchema);
- int[] columnCardinality = new int[] { 1, 2 };
- new MockUp<ColumnSchema>() {
- @Mock public List<Encoding> getEncodingList() {
- List<Encoding> encodingList = new ArrayList<>();
- encodingList.add(Encoding.DICTIONARY);
- return encodingList;
- }
- };
- SegmentProperties segmentProperties = new SegmentProperties(columnsInTable, columnCardinality);
- assertTrue(FilterUtil.prepareDefaultStartIndexKey(segmentProperties) instanceof IndexKey);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scan/result/BatchResultTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/result/BatchResultTest.java b/core/src/test/java/org/apache/carbondata/scan/result/BatchResultTest.java
deleted file mode 100644
index 5a6225e..0000000
--- a/core/src/test/java/org/apache/carbondata/scan/result/BatchResultTest.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.result;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.NoSuchElementException;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-
-public class BatchResultTest {
- private static BatchResult batchResult;
- private static List<Object[]> rowsList = new ArrayList(2);
-
- @BeforeClass public static void setUp() {
- batchResult = new BatchResult();
- rowsList.add(0, new Integer[] { 1, 2 });
- rowsList.add(1, new Integer[] { 3 });
- }
-
- @Test public void testNext() throws NoSuchElementException {
- BatchResult rows = new BatchResult();
- rows.setRows(rowsList);
- Object[] result = rows.next();
- assert (result.equals(rowsList.get(0)));
- }
-
- @Test(expected = NoSuchElementException.class) public void testNextWithNoSuchElementException() {
- BatchResult rows = new BatchResult();
- List emptyList = new ArrayList(2);
- rows.setRows(emptyList);
- rows.next();
- }
-
- @Test public void testGetRows() {
- new MockUp<BatchResult>() {
- @Mock public void $init() {
- //to be left blank
- }
- };
- BatchResult batchResult = new BatchResult();
- List<Object[]> list = batchResult.getRows();
- assertNull("Number of rows is null", list);
- }
-
- @Test public void testHasNext() {
- List<Object[]> list = new ArrayList<>();
- list.add(0, new Integer[] { 1, 2 });
- list.add(1, new Integer[] { 1, 2 });
- batchResult.setRows(list);
- boolean result = batchResult.hasNext();
- assert (result);
- }
-
- @Test public void testGetRawRow() {
- List<Object[]> list = new ArrayList<>();
- list.add(0, new Integer[] { 1, 2 });
- batchResult.setRows(list);
- Object[] actualValue = batchResult.getRawRow(0);
- assert (list.get(0) == actualValue);
- }
-
- @Test public void testGetSize() {
- List<Object[]> list = new ArrayList<>();
- list.add(0, new Integer[] { 1, 2 });
- list.add(1, new Integer[] { 1, 2 });
- batchResult.setRows(list);
- int actualValue = batchResult.getSize();
- int expectedValue = 2;
- assertEquals(expectedValue, actualValue);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scan/result/impl/FilterQueryScannedResultTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/result/impl/FilterQueryScannedResultTest.java b/core/src/test/java/org/apache/carbondata/scan/result/impl/FilterQueryScannedResultTest.java
deleted file mode 100644
index 9aabaeb..0000000
--- a/core/src/test/java/org/apache/carbondata/scan/result/impl/FilterQueryScannedResultTest.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.result.impl;
-
-
-public class FilterQueryScannedResultTest {
-
-// private static FilterQueryScannedResult filterQueryScannedResult;
-//
-// @BeforeClass public static void setUp() {
-// BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
-// blockExecutionInfo.setFixedLengthKeySize(2);
-// blockExecutionInfo.setNoDictionaryBlockIndexes(new int[] { 0, 1 });
-// blockExecutionInfo.setDictionaryColumnBlockIndex(new int[] { 0, 1 });
-// Map<Integer, KeyStructureInfo> columnGourpToKeyInfo = new HashMap<>();
-// columnGourpToKeyInfo.put(1, new KeyStructureInfo());
-// blockExecutionInfo.setColumnGroupToKeyStructureInfo(columnGourpToKeyInfo);
-// Map<Integer, GenericQueryType> genericQueryType = new HashMap<>();
-// genericQueryType.put(1, new ArrayQueryType("Query1", "Parent", 1));
-// blockExecutionInfo.setComplexDimensionInfoMap(genericQueryType);
-// blockExecutionInfo.setComplexColumnParentBlockIndexes(new int[] { 1 });
-// QueryDimension[] queryDimensions = { new QueryDimension("Col1"), new QueryDimension("Col2") };
-// blockExecutionInfo.setQueryDimensions(queryDimensions);
-// filterQueryScannedResult = new FilterQueryScannedResult(blockExecutionInfo);
-// filterQueryScannedResult.setIndexes(new int[] { 1, 2, 3, 4 });
-// DimensionChunkAttributes dimensionChunkAttributes = new DimensionChunkAttributes();
-// dimensionChunkAttributes.setEachRowSize(0);
-// ColumnGroupDimensionDataChunk[] columnGroupDimensionDataChunks =
-// { new ColumnGroupDimensionDataChunk(new byte[] { 1, 2 }, dimensionChunkAttributes),
-// new ColumnGroupDimensionDataChunk(new byte[] { 2, 3 }, dimensionChunkAttributes) };
-// filterQueryScannedResult.setDimensionChunks(columnGroupDimensionDataChunks);
-// MeasureColumnDataChunk measureColumnDataChunk = new MeasureColumnDataChunk();
-// filterQueryScannedResult
-// .setMeasureChunks(new MeasureColumnDataChunk[] { measureColumnDataChunk });
-// }
-//
-// @Test public void testToGetDictionaryKeyArray() {
-// new MockUp<ColumnGroupDimensionDataChunk>() {
-// @Mock @SuppressWarnings("unused") public int fillChunkData(byte[] data, int offset, int rowId,
-// KeyStructureInfo restructuringInfo) {
-// return 1;
-// }
-// };
-// byte[] keyArray = filterQueryScannedResult.getDictionaryKeyArray();
-// byte[] expectedResult = { 0, 0 };
-// assertThat(expectedResult, is(equalTo(keyArray)));
-// }
-//
-// @Test public void testToGetDictionaryKeyIntegerArray() {
-// new MockUp<ColumnGroupDimensionDataChunk>() {
-// @Mock @SuppressWarnings("unused")
-// public int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
-// KeyStructureInfo info) {
-// return 1;
-// }
-// };
-// int[] keyArray = filterQueryScannedResult.getDictionaryKeyIntegerArray();
-// int[] expectedResult = { 0, 0 };
-// assertThat(expectedResult, is(equalTo(keyArray)));
-// }
-//
-// @Test public void testToGetComplexTypeKeyArray() {
-// new MockUp<ByteArrayOutputStream>() {
-// @Mock @SuppressWarnings("unused") public synchronized byte toByteArray()[] {
-// return new byte[] { 1, 2, 3 };
-// }
-// };
-// new MockUp<ArrayQueryType>() {
-// @Mock @SuppressWarnings("unused") public void parseBlocksAndReturnComplexColumnByteArray(
-// DimensionColumnDataChunk[] dimensionColumnDataChunks, int rowNumber,
-// DataOutputStream dataOutputStream) throws IOException {
-// }
-// };
-// filterQueryScannedResult.incrementCounter();
-// byte[][] keyArray = filterQueryScannedResult.getComplexTypeKeyArray();
-// byte[][] expectedResult = { { 1, 2, 3 } };
-// assertThat(expectedResult, is(equalTo(keyArray)));
-// }
-//
-// @Test public void testToGetNoDictionaryKeyArray() {
-// new MockUp<ColumnGroupDimensionDataChunk>() {
-// @Mock @SuppressWarnings("unused") public byte[] getChunkData(int rowId) {
-// return new byte[] { 1, 2, 3 };
-// }
-// };
-// byte[][] dictionaryKeyArray = filterQueryScannedResult.getNoDictionaryKeyArray();
-// byte[][] expectedResult = { { 1, 2, 3 }, { 1, 2, 3 } };
-// assertThat(expectedResult, is(equalTo(dictionaryKeyArray)));
-// }
-//
-// @Test public void testToGetNoDictionaryKeyStringArray() {
-// new MockUp<ColumnGroupDimensionDataChunk>() {
-// @Mock @SuppressWarnings("unused") public byte[] getChunkData(int rowId) {
-// return "1".getBytes();
-// }
-// };
-// filterQueryScannedResult.incrementCounter();
-// String[] dictionaryKeyStringArray = filterQueryScannedResult.getNoDictionaryKeyStringArray();
-// String[] expectedResult = { "1", "1" };
-// assertThat(expectedResult, is(equalTo(dictionaryKeyStringArray)));
-// }
-//
-// @Test public void testToGetCurrenrRowId() {
-// int rowId = filterQueryScannedResult.getCurrenrRowId();
-// int expectedResult = 3;
-// assertThat(expectedResult, is(equalTo(rowId)));
-// }
-//
-// @Test public void testToGetDimensionKey() {
-// new MockUp<ColumnGroupDimensionDataChunk>() {
-// @Mock @SuppressWarnings("unused") public byte[] getChunkData(int rowId) {
-// return "1".getBytes();
-// }
-// };
-// byte[] dictionaryKeyStringArray = filterQueryScannedResult.getDimensionKey(0);
-// byte[] expectedResult = "1".getBytes();
-// assertThat(expectedResult, is(equalTo(dictionaryKeyStringArray)));
-// }
-//
-// @Test public void testToGetIsNullMeasureValue() {
-// new MockUp<MeasureColumnDataChunk>() {
-// @Mock @SuppressWarnings("unused") public PresenceMeta getNullValueIndexHolder() {
-// return new PresenceMeta();
-//
-// }
-// };
-// new MockUp<PresenceMeta>() {
-// @Mock @SuppressWarnings("unused") public BitSet getBitSet() {
-// return new BitSet();
-// }
-// };
-// new MockUp<BitSet>() {
-// @Mock @SuppressWarnings("unused") public boolean get(int bitIndex) {
-// return false;
-// }
-// };
-//
-// boolean nullMeasureValue = filterQueryScannedResult.isNullMeasureValue(0);
-// assertThat(false, is(equalTo(nullMeasureValue)));
-// }
-//
-// @Test public void testToGetLongMeasureValue() {
-// new MockUp<MeasureColumnDataChunk>() {
-// @Mock @SuppressWarnings("unused") public CarbonReadDataHolder getMeasureDataHolder() {
-// return new CarbonReadDataHolder();
-//
-// }
-// };
-// new MockUp<CarbonReadDataHolder>() {
-// @Mock @SuppressWarnings("unused") public long getReadableLongValueByIndex(int index) {
-// return 2L;
-// }
-// };
-// long longMeasureValue = filterQueryScannedResult.getLongMeasureValue(0);
-// long expectedResult = 2L;
-// assertThat(expectedResult, is(equalTo(longMeasureValue)));
-// }
-//
-// @Test public void testToGetDoubleMeasureValue() {
-// new MockUp<MeasureColumnDataChunk>() {
-// @Mock @SuppressWarnings("unused") public CarbonReadDataHolder getMeasureDataHolder() {
-// return new CarbonReadDataHolder();
-//
-// }
-// };
-// new MockUp<CarbonReadDataHolder>() {
-// @Mock @SuppressWarnings("unused") public double getReadableDoubleValueByIndex(int index) {
-// return 2.0;
-// }
-// };
-// double longMeasureValue = filterQueryScannedResult.getDoubleMeasureValue(0);
-// double expectedResult = 2.0;
-// assertThat(expectedResult, is(equalTo(longMeasureValue)));
-// }
-//
-// @Test public void testToGetBigDecimalMeasureValue() {
-// new MockUp<MeasureColumnDataChunk>() {
-// @Mock @SuppressWarnings("unused") public CarbonReadDataHolder getMeasureDataHolder() {
-// return new CarbonReadDataHolder();
-//
-// }
-// };
-// new MockUp<CarbonReadDataHolder>() {
-// @Mock @SuppressWarnings("unused")
-// public BigDecimal getReadableBigDecimalValueByIndex(int index) {
-// return new BigDecimal(2);
-// }
-// };
-// BigDecimal longMeasureValue = filterQueryScannedResult.getBigDecimalMeasureValue(0);
-// BigDecimal expectedResult = new BigDecimal(2);
-// assertThat(expectedResult, is(equalTo(longMeasureValue)));
-// }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scan/result/impl/NonFilterQueryScannedResultTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/result/impl/NonFilterQueryScannedResultTest.java b/core/src/test/java/org/apache/carbondata/scan/result/impl/NonFilterQueryScannedResultTest.java
deleted file mode 100644
index 09f0b75..0000000
--- a/core/src/test/java/org/apache/carbondata/scan/result/impl/NonFilterQueryScannedResultTest.java
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.result.impl;
-
-
-public class NonFilterQueryScannedResultTest {
-
-// private static NonFilterQueryScannedResult filterQueryScannedResult;
-//
-// @BeforeClass public static void setUp() {
-// BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
-// blockExecutionInfo.setFixedLengthKeySize(2);
-// blockExecutionInfo.setNoDictionaryBlockIndexes(new int[] { 0, 1 });
-// blockExecutionInfo.setDictionaryColumnBlockIndex(new int[] { 0, 1 });
-// Map<Integer, KeyStructureInfo> columnGourpToKeyInfo = new HashMap<>();
-// columnGourpToKeyInfo.put(1, new KeyStructureInfo());
-// blockExecutionInfo.setColumnGroupToKeyStructureInfo(columnGourpToKeyInfo);
-// Map<Integer, GenericQueryType> genericQueryType = new HashMap<>();
-// genericQueryType.put(1, new ArrayQueryType("Query1", "Parent", 1));
-// blockExecutionInfo.setComplexDimensionInfoMap(genericQueryType);
-// blockExecutionInfo.setComplexColumnParentBlockIndexes(new int[] { 1 });
-// QueryDimension[] queryDimensions = { new QueryDimension("Col1"), new QueryDimension("Col2") };
-// blockExecutionInfo.setQueryDimensions(queryDimensions);
-// filterQueryScannedResult = new NonFilterQueryScannedResult(blockExecutionInfo);
-// DimensionChunkAttributes dimensionChunkAttributes = new DimensionChunkAttributes();
-// dimensionChunkAttributes.setEachRowSize(0);
-// ColumnGroupDimensionDataChunk[] columnGroupDimensionDataChunks =
-// { new ColumnGroupDimensionDataChunk(new byte[] { 1, 2 }, dimensionChunkAttributes),
-// new ColumnGroupDimensionDataChunk(new byte[] { 2, 3 }, dimensionChunkAttributes) };
-// filterQueryScannedResult.setDimensionChunks(columnGroupDimensionDataChunks);
-// MeasureColumnDataChunk measureColumnDataChunk = new MeasureColumnDataChunk();
-// filterQueryScannedResult
-// .setMeasureChunks(new MeasureColumnDataChunk[] { measureColumnDataChunk });
-// }
-//
-// @Test public void testToGetDictionaryKeyArray() {
-// new MockUp<ColumnGroupDimensionDataChunk>() {
-// @Mock @SuppressWarnings("unused") public int fillChunkData(byte[] data, int offset, int rowId,
-// KeyStructureInfo restructuringInfo) {
-// return 1;
-// }
-// };
-// byte[] keyArray = filterQueryScannedResult.getDictionaryKeyArray();
-// byte[] expectedResult = { 0, 0 };
-// assertThat(expectedResult, is(equalTo(keyArray)));
-// }
-//
-// @Test public void testToGetDictionaryKeyIntegerArray() {
-// new MockUp<ColumnGroupDimensionDataChunk>() {
-// @Mock @SuppressWarnings("unused")
-// public int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
-// KeyStructureInfo info) {
-// return 1;
-// }
-// };
-// int[] keyArray = filterQueryScannedResult.getDictionaryKeyIntegerArray();
-// int[] expectedResult = { 0, 0 };
-// assertThat(expectedResult, is(equalTo(keyArray)));
-// }
-//
-// @Test public void testToGetComplexTypeKeyArray() {
-// new MockUp<ByteArrayOutputStream>() {
-// @Mock @SuppressWarnings("unused") public synchronized byte toByteArray()[] {
-// return new byte[] { 1, 2, 3 };
-// }
-// };
-// new MockUp<ArrayQueryType>() {
-// @Mock @SuppressWarnings("unused") public void parseBlocksAndReturnComplexColumnByteArray(
-// DimensionColumnDataChunk[] dimensionColumnDataChunks, int rowNumber,
-// DataOutputStream dataOutputStream) throws IOException {
-// }
-// };
-// filterQueryScannedResult.incrementCounter();
-// byte[][] keyArray = filterQueryScannedResult.getComplexTypeKeyArray();
-// byte[][] expectedResult = { { 1, 2, 3 } };
-// assertThat(expectedResult, is(equalTo(keyArray)));
-// }
-//
-// @Test public void testToGetNoDictionaryKeyArray() {
-// new MockUp<ColumnGroupDimensionDataChunk>() {
-// @Mock @SuppressWarnings("unused") public byte[] getChunkData(int rowId) {
-// return new byte[] { 1, 2, 3 };
-// }
-// };
-// byte[][] dictionaryKeyArray = filterQueryScannedResult.getNoDictionaryKeyArray();
-// byte[][] expectedResult = { { 1, 2, 3 }, { 1, 2, 3 } };
-// assertThat(expectedResult, is(equalTo(dictionaryKeyArray)));
-// }
-//
-// @Test public void testToGetNoDictionaryKeyStringArray() {
-// new MockUp<ColumnGroupDimensionDataChunk>() {
-// @Mock @SuppressWarnings("unused") public byte[] getChunkData(int rowId) {
-// return "1".getBytes();
-// }
-// };
-// filterQueryScannedResult.incrementCounter();
-// String[] dictionaryKeyStringArray = filterQueryScannedResult.getNoDictionaryKeyStringArray();
-// String[] expectedResult = { "1", "1" };
-// assertThat(expectedResult, is(equalTo(dictionaryKeyStringArray)));
-// }
-//
-// @Test public void testToGetCurrenrRowId() {
-// int rowId = filterQueryScannedResult.getCurrenrRowId();
-// int expectedResult = 2;
-// assertThat(expectedResult, is(equalTo(rowId)));
-// }
-//
-// @Test public void testToGetDimensionKey() {
-// new MockUp<ColumnGroupDimensionDataChunk>() {
-// @Mock @SuppressWarnings("unused") public byte[] getChunkData(int rowId) {
-// return "1".getBytes();
-// }
-// };
-// byte[] dictionaryKeyStringArray = filterQueryScannedResult.getDimensionKey(0);
-// byte[] expectedResult = "1".getBytes();
-// assertThat(expectedResult, is(equalTo(dictionaryKeyStringArray)));
-// }
-//
-// @Test public void testToGetIsNullMeasureValue() {
-// new MockUp<MeasureColumnDataChunk>() {
-// @Mock @SuppressWarnings("unused") public PresenceMeta getNullValueIndexHolder() {
-// return new PresenceMeta();
-//
-// }
-// };
-// new MockUp<PresenceMeta>() {
-// @Mock @SuppressWarnings("unused") public BitSet getBitSet() {
-// return new BitSet();
-// }
-// };
-// new MockUp<BitSet>() {
-// @Mock @SuppressWarnings("unused") public boolean get(int bitIndex) {
-// return false;
-// }
-// };
-//
-// boolean nullMeasureValue = filterQueryScannedResult.isNullMeasureValue(0);
-// assertThat(false, is(equalTo(nullMeasureValue)));
-// }
-//
-// @Test public void testToGetLongMeasureValue() {
-// new MockUp<MeasureColumnDataChunk>() {
-// @Mock @SuppressWarnings("unused") public CarbonReadDataHolder getMeasureDataHolder() {
-// return new CarbonReadDataHolder();
-//
-// }
-// };
-// new MockUp<CarbonReadDataHolder>() {
-// @Mock @SuppressWarnings("unused") public long getReadableLongValueByIndex(int index) {
-// return 2L;
-// }
-// };
-// long longMeasureValue = filterQueryScannedResult.getLongMeasureValue(0);
-// long expectedResult = 2L;
-// assertThat(expectedResult, is(equalTo(longMeasureValue)));
-// }
-//
-// @Test public void testToGetDoubleMeasureValue() {
-// new MockUp<MeasureColumnDataChunk>() {
-// @Mock @SuppressWarnings("unused") public CarbonReadDataHolder getMeasureDataHolder() {
-// return new CarbonReadDataHolder();
-//
-// }
-// };
-// new MockUp<CarbonReadDataHolder>() {
-// @Mock @SuppressWarnings("unused") public double getReadableDoubleValueByIndex(int index) {
-// return 2.0;
-// }
-// };
-// double longMeasureValue = filterQueryScannedResult.getDoubleMeasureValue(0);
-// double expectedResult = 2.0;
-// assertThat(expectedResult, is(equalTo(longMeasureValue)));
-// }
-//
-// @Test public void testToGetBigDecimalMeasureValue() {
-// new MockUp<MeasureColumnDataChunk>() {
-// @Mock @SuppressWarnings("unused") public CarbonReadDataHolder getMeasureDataHolder() {
-// return new CarbonReadDataHolder();
-//
-// }
-// };
-// new MockUp<CarbonReadDataHolder>() {
-// @Mock @SuppressWarnings("unused")
-// public BigDecimal getReadableBigDecimalValueByIndex(int index) {
-// return new BigDecimal(2);
-// }
-// };
-// BigDecimal longMeasureValue = filterQueryScannedResult.getBigDecimalMeasureValue(0);
-// BigDecimal expectedResult = new BigDecimal(2);
-// assertThat(expectedResult, is(equalTo(longMeasureValue)));
-// }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scan/wrappers/ByteArrayWrapperTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scan/wrappers/ByteArrayWrapperTest.java b/core/src/test/java/org/apache/carbondata/scan/wrappers/ByteArrayWrapperTest.java
deleted file mode 100644
index dfeb63d..0000000
--- a/core/src/test/java/org/apache/carbondata/scan/wrappers/ByteArrayWrapperTest.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.wrappers;
-
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static junit.framework.TestCase.assertEquals;
-import static junit.framework.TestCase.assertFalse;
-import static junit.framework.TestCase.assertTrue;
-
-public class ByteArrayWrapperTest {
- private static ByteArrayWrapper byteArrayWrapper;
- byte[] dictionaryKey = new byte[] { 1 };
- byte[][] complexTypesKeys = { { 1 }, { 1 } };
- byte[][] noDictionaryKeys = new byte[][] { { 1 }, { 1 } };
-
- @BeforeClass
-
- public static void setUp() {
- byteArrayWrapper = new ByteArrayWrapper();
- }
-
- @AfterClass public static void tearDown() {
- byteArrayWrapper = null;
- }
-
- @Test public void testHashCodeValue() {
- byteArrayWrapper.setDictionaryKey(dictionaryKey);
- byteArrayWrapper.setComplexTypesKeys(complexTypesKeys);
- byteArrayWrapper.setNoDictionaryKeys(noDictionaryKeys);
-
- int result = byteArrayWrapper.hashCode();
- assert (29583456 == result);
- }
-
- @Test public void testEqualsWithOtherAsInstanceOfByteArrayWrapper() {
- ByteArrayWrapper other;
- other = null;
- boolean result = byteArrayWrapper.equals(other);
- assertFalse(result);
- }
-
- @Test public void testEqualsWithNoDictionaryKeysOtherNotEqualNoDictionaryKeys() {
- byte[][] noDictionaryKeysOther = { { 1 } };
- byteArrayWrapper.setNoDictionaryKeys(noDictionaryKeys);
- ByteArrayWrapper other = new ByteArrayWrapper();
- other.setNoDictionaryKeys(noDictionaryKeysOther);
- boolean result = byteArrayWrapper.equals(other);
- assertFalse(result);
- }
-
- @Test public void testEqualsWithComplexTypesKeysOtherNotEqualComplexTypesKeys() {
- byte[][] complexTypesKeysOther = { { 1 } };
- byteArrayWrapper.setComplexTypesKeys(complexTypesKeys);
- byteArrayWrapper.setNoDictionaryKeys(noDictionaryKeys);
- ByteArrayWrapper other = new ByteArrayWrapper();
- other.setComplexTypesKeys(complexTypesKeysOther);
- other.setNoDictionaryKeys(noDictionaryKeys);
- boolean result = byteArrayWrapper.equals(other);
- assertFalse(result);
- }
-
- @Test
-
- public void testEqualsForFirstElementComplexTypesKeysAndOther() {
- ByteArrayWrapper other = new ByteArrayWrapper();
- other.setComplexTypesKeys(complexTypesKeys);
- other.setDictionaryKey(dictionaryKey);
- other.setNoDictionaryKeys(noDictionaryKeys);
- byteArrayWrapper.setComplexTypesKeys(complexTypesKeys);
- byteArrayWrapper.setDictionaryKey(dictionaryKey);
- byteArrayWrapper.setNoDictionaryKeys(noDictionaryKeys);
- boolean result = byteArrayWrapper.equals(other);
- assertTrue(result);
- }
-
- @Test public void testCompareTo() {
- byteArrayWrapper.setDictionaryKey(dictionaryKey);
- ByteArrayWrapper other = new ByteArrayWrapper();
- other.setDictionaryKey(dictionaryKey);
- other.setNoDictionaryKeys(noDictionaryKeys);
- other.setComplexTypesKeys(complexTypesKeys);
- byteArrayWrapper.setNoDictionaryKeys(noDictionaryKeys);
- byteArrayWrapper.setComplexTypesKeys(complexTypesKeys);
- int actualResult = byteArrayWrapper.compareTo(other);
- int expectedResult = 0;
- assertEquals(expectedResult, actualResult);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/scanner/impl/FilterScannerTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/scanner/impl/FilterScannerTest.java b/core/src/test/java/org/apache/carbondata/scanner/impl/FilterScannerTest.java
deleted file mode 100644
index 6e8534a..0000000
--- a/core/src/test/java/org/apache/carbondata/scanner/impl/FilterScannerTest.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scanner.impl;
-
-import static junit.framework.TestCase.assertEquals;
-
-public class FilterScannerTest {
-//
-// private static FilterScanner filterScanner;
-// private static BlockletIndex blockletIndex;
-// private static BlockletMinMaxIndex blockletMinMaxIndex;
-// private static BTreeBuilderInfo bTreeBuilderInfo;
-// private static DataFileFooter dataFileFooter;
-//
-// @BeforeClass public static void setUp() {
-// BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
-// FilterExecuter filterExecutor = new AndFilterExecuterImpl(null, null);
-// blockExecutionInfo.setFilterExecuterTree(filterExecutor);
-// blockExecutionInfo.setFixedLengthKeySize(1);
-// blockExecutionInfo.setNoDictionaryBlockIndexes(new int[] { 1, 2 });
-// blockExecutionInfo.setDictionaryColumnBlockIndex(new int[] { 1 });
-// blockExecutionInfo.setColumnGroupToKeyStructureInfo(new HashMap<Integer, KeyStructureInfo>());
-// blockExecutionInfo.setComplexDimensionInfoMap(new HashMap<Integer, GenericQueryType>());
-// blockExecutionInfo.setComplexColumnParentBlockIndexes(new int[] { 1 });
-// blockExecutionInfo.setQueryDimensions(new QueryDimension[] { new QueryDimension("Col1") });
-// blockExecutionInfo.setAllSelectedDimensionBlocksIndexes(new int[][] { { 0, 0 } });
-// blockExecutionInfo.setAllSelectedMeasureBlocksIndexes(new int[][] { { 0, 0 } });
-// blockExecutionInfo.setTotalNumberOfMeasureBlock(1);
-// blockExecutionInfo.setTotalNumberDimensionBlock(1);
-// QueryStatisticsModel queryStatisticsModel = new QueryStatisticsModel();
-// QueryStatistic queryStatistic = new QueryStatistic();
-// queryStatistic.addCountStatistic(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM, 1);
-// Map<String, QueryStatistic> statisticsTypeAndObjMap = new HashMap<>();
-// statisticsTypeAndObjMap.put(QueryStatisticsConstants.TOTAL_BLOCKLET_NUM, queryStatistic);
-// statisticsTypeAndObjMap.put(QueryStatisticsConstants.VALID_SCAN_BLOCKLET_NUM, queryStatistic);
-// queryStatisticsModel.setStatisticsTypeAndObjMap(statisticsTypeAndObjMap);
-// QueryStatisticsRecorder queryStatisticsRecorder = new QueryStatisticsRecorderImpl("1");
-// queryStatisticsModel.setRecorder(queryStatisticsRecorder);
-// filterScanner = new FilterScanner(blockExecutionInfo, queryStatisticsModel);
-// blockletIndex = new BlockletIndex();
-// blockletMinMaxIndex = new BlockletMinMaxIndex();
-// blockletMinMaxIndex.setMinValues(new byte[][] { { 1, 2 } });
-// blockletMinMaxIndex.setMaxValues(new byte[][] { { 10, 12 } });
-// blockletIndex.setMinMaxIndex(blockletMinMaxIndex);
-// dataFileFooter = new DataFileFooter();
-// dataFileFooter.setBlockletIndex(blockletIndex);
-// bTreeBuilderInfo = new BTreeBuilderInfo(Arrays.asList(dataFileFooter), new int[] { 1 });
-// }
-//
-// @Test public void testToScanBlockletWithEmptyBitSet() throws QueryExecutionException {
-// new MockUp<AndFilterExecuterImpl>() {
-// @SuppressWarnings("unused") @Mock
-// public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
-// return new BitSet();
-// }
-// };
-// BlocksChunkHolder blocksChunkHolder = new BlocksChunkHolder(1, 1);
-// DataRefNode dataRefNode = new BlockBTreeLeafNode(bTreeBuilderInfo, 0, 1);
-// blocksChunkHolder.setDataBlock(dataRefNode);
-// AbstractScannedResult abstractScannedResult = filterScanner.scanBlocklet(blocksChunkHolder);
-// assertEquals(0, abstractScannedResult.numberOfOutputRows());
-// }
-//
-// @Test public void testToScanBlockletWithNonEmptyBitSet() throws QueryExecutionException {
-// new MockUp<AndFilterExecuterImpl>() {
-// @SuppressWarnings("unused") @Mock
-// public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
-// BitSet bitSet = new BitSet();
-// bitSet.set(1);
-// bitSet.set(2);
-// bitSet.set(1);
-// return bitSet;
-// }
-//
-// @SuppressWarnings("unused") @Mock
-// public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
-// throws FilterUnsupportedException {
-// BitSet bitSet = new BitSet();
-// bitSet.set(1);
-// bitSet.set(2);
-// bitSet.set(1);
-// return bitSet;
-// }
-// };
-// DataRefNode dataRefNode = new MockUp<DataRefNode>() {
-// @Mock @SuppressWarnings("unused") DimensionColumnDataChunk[] getDimensionChunks(
-// FileHolder fileReader, int[][] blockIndexes) {
-// DimensionColumnDataChunk[] dimensionChunkAttributes =
-// { new ColumnGroupDimensionDataChunk(null, null) };
-// return dimensionChunkAttributes;
-// }
-//
-// @Mock @SuppressWarnings("unused") MeasureColumnDataChunk[] getMeasureChunks(
-// FileHolder fileReader, int[][] blockIndexes) {
-//
-// MeasureColumnDataChunk[] measureColumnDataChunks = { new MeasureColumnDataChunk() };
-// return measureColumnDataChunks;
-// }
-// }.getMockInstance();
-//
-// BlocksChunkHolder blocksChunkHolder = new BlocksChunkHolder(1, 1);
-// blocksChunkHolder.setDataBlock(dataRefNode);
-// DimensionChunkAttributes dimensionChunkAttributes = new DimensionChunkAttributes();
-// DimensionColumnDataChunk dimensionColumnDataChunk =
-// new FixedLengthDimensionDataChunk(new byte[] { 0, 1 }, dimensionChunkAttributes);
-// blocksChunkHolder.setDimensionDataChunk(new DimensionColumnDataChunk[]
-//
-// { dimensionColumnDataChunk });
-// MeasureColumnDataChunk measureColumnDataChunk = new MeasureColumnDataChunk();
-// blocksChunkHolder.setMeasureDataChunk(new MeasureColumnDataChunk[]
-//
-// { measureColumnDataChunk });
-// FileHolder fileHolder = new DFSFileHolderImpl();
-// blocksChunkHolder.setFileReader(fileHolder);
-// AbstractScannedResult abstractScannedResult = filterScanner.scanBlocklet(blocksChunkHolder);
-//
-// assertEquals(2, abstractScannedResult.numberOfOutputRows());
-// }
-//
-// @Test(expected = QueryExecutionException.class) public void testToScanBlockletWithException()
-// throws QueryExecutionException {
-// new MockUp<AndFilterExecuterImpl>() {
-// @SuppressWarnings("unused") @Mock
-// public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
-// BitSet bitSet = new BitSet();
-// bitSet.set(1);
-// bitSet.set(2);
-// bitSet.set(1);
-// return bitSet;
-// }
-//
-// @SuppressWarnings("unused") @Mock
-// public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
-// throws FilterUnsupportedException {
-// throw new FilterUnsupportedException("Filter unsupported");
-// }
-// };
-// BlocksChunkHolder blocksChunkHolder = new BlocksChunkHolder(1, 1);
-// BTreeBuilderInfo bTreeBuilderInfo =
-// new BTreeBuilderInfo(Arrays.asList(dataFileFooter), new int[] { 1 });
-// DataRefNode dataRefNode = new BlockBTreeLeafNode(bTreeBuilderInfo, 0, 1);
-// blocksChunkHolder.setDataBlock(dataRefNode);
-// filterScanner.scanBlocklet(blocksChunkHolder);
-// }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/dev/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/dev/findbugs-exclude.xml b/dev/findbugs-exclude.xml
index be54b56..90239ad 100644
--- a/dev/findbugs-exclude.xml
+++ b/dev/findbugs-exclude.xml
@@ -34,12 +34,12 @@
</Match>
<Match>
- <Class name="org.apache.carbondata.scan.aggregator.impl.BitSet"/>
+ <Class name="org.apache.carbondata.core.scan.aggregator.impl.BitSet"/>
<Bug pattern="SE_TRANSIENT_FIELD_NOT_RESTORED"/>
</Match>
<Match>
- <Class name="org.apache.carbondata.scan.expression.ExpressionResult"/>
+ <Class name="org.apache.carbondata.core.scan.expression.ExpressionResult"/>
<Or>
<Method name="getBoolean"/>
</Or>
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/examples/spark/src/main/scala/org/apache/carbondata/examples/AlluxioExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/AlluxioExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/AlluxioExample.scala
index 17fe960..963552e 100644
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/AlluxioExample.scala
+++ b/examples/spark/src/main/scala/org/apache/carbondata/examples/AlluxioExample.scala
@@ -18,7 +18,7 @@
package org.apache.carbondata.examples
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory
+import org.apache.carbondata.core.datastorage.impl.FileFactory
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.examples.util.ExampleUtils
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/examples/spark/src/main/scala/org/apache/carbondata/examples/util/AllDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/util/AllDictionaryUtil.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/util/AllDictionaryUtil.scala
index 6011bcb..e16e084 100644
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/util/AllDictionaryUtil.scala
+++ b/examples/spark/src/main/scala/org/apache/carbondata/examples/util/AllDictionaryUtil.scala
@@ -24,7 +24,7 @@ import scala.collection.mutable.{ArrayBuffer, HashSet}
import org.apache.spark.SparkContext
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory
+import org.apache.carbondata.core.datastorage.impl.FileFactory
object AllDictionaryUtil {
private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName)
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
index 615629d..23429c4 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
@@ -22,7 +22,7 @@ import java.io.IOException;
import java.lang.reflect.Constructor;
import java.util.*;
-import org.apache.carbondata.common.iudprocessor.iuddata.BlockMappingVO;
+import org.apache.carbondata.core.update.data.BlockMappingVO;
import org.apache.carbondata.core.AbsoluteTableIdentifier;
import org.apache.carbondata.core.ColumnarFormatVersion;
import org.apache.carbondata.core.datastore.DataRefNode;
@@ -41,7 +41,7 @@ import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.stats.QueryStatistic;
import org.apache.carbondata.core.stats.QueryStatisticsConstants;
import org.apache.carbondata.core.stats.QueryStatisticsRecorder;
-import org.apache.carbondata.core.update.UpdateVO;
+import org.apache.carbondata.core.update.data.UpdateVO;
import org.apache.carbondata.core.updatestatus.SegmentStatusManager;
import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager;
import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
@@ -52,12 +52,12 @@ import org.apache.carbondata.hadoop.util.BlockLevelTraverser;
import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil;
import org.apache.carbondata.hadoop.util.ObjectSerializationUtil;
import org.apache.carbondata.hadoop.util.SchemaReader;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.filter.FilterExpressionProcessor;
-import org.apache.carbondata.scan.filter.FilterUtil;
-import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
-import org.apache.carbondata.scan.model.CarbonQueryPlan;
-import org.apache.carbondata.scan.model.QueryModel;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.filter.FilterExpressionProcessor;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+import org.apache.carbondata.core.scan.model.CarbonQueryPlan;
+import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
index 36bf3a1..a42995f 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
@@ -33,7 +33,7 @@ import org.apache.carbondata.core.datastore.block.Distributable;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.update.UpdateVO;
+import org.apache.carbondata.core.update.data.UpdateVO;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.hadoop.internal.index.Block;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
index 9c17f5e..af6611f 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonRecordReader.java
@@ -28,11 +28,11 @@ import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.hadoop.readsupport.CarbonReadSupport;
-import org.apache.carbondata.scan.executor.QueryExecutor;
-import org.apache.carbondata.scan.executor.QueryExecutorFactory;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
-import org.apache.carbondata.scan.model.QueryModel;
-import org.apache.carbondata.scan.result.iterator.ChunkRowIterator;
+import org.apache.carbondata.core.scan.executor.QueryExecutor;
+import org.apache.carbondata.core.scan.executor.QueryExecutorFactory;
+import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
+import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.core.scan.result.iterator.ChunkRowIterator;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index f353b55..0eca474 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -30,8 +30,8 @@ import org.apache.carbondata.hadoop.internal.segment.SegmentManager;
import org.apache.carbondata.hadoop.internal.segment.SegmentManagerFactory;
import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil;
import org.apache.carbondata.hadoop.util.ObjectSerializationUtil;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.InputSplit;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/Index.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/Index.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/Index.java
index 49404f7..0e2f3f6 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/Index.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/Index.java
@@ -22,7 +22,7 @@ package org.apache.carbondata.hadoop.internal.index;
import java.io.IOException;
import java.util.List;
-import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.hadoop.mapreduce.JobContext;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
index 8a2ce37..a036b2a 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
@@ -49,9 +49,9 @@ import org.apache.carbondata.hadoop.CarbonInputSplit;
import org.apache.carbondata.hadoop.internal.index.Block;
import org.apache.carbondata.hadoop.internal.index.Index;
import org.apache.carbondata.hadoop.internal.segment.Segment;
-import org.apache.carbondata.scan.filter.FilterExpressionProcessor;
-import org.apache.carbondata.scan.filter.FilterUtil;
-import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
+import org.apache.carbondata.core.scan.filter.FilterExpressionProcessor;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/Segment.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/Segment.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/Segment.java
index 067b23f..6a99bd7 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/Segment.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/Segment.java
@@ -23,7 +23,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/impl/IndexedSegment.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/impl/IndexedSegment.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/impl/IndexedSegment.java
index 76c1415..0215e26 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/impl/IndexedSegment.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/segment/impl/IndexedSegment.java
@@ -27,7 +27,7 @@ import org.apache.carbondata.hadoop.internal.index.Block;
import org.apache.carbondata.hadoop.internal.index.Index;
import org.apache.carbondata.hadoop.internal.index.IndexLoader;
import org.apache.carbondata.hadoop.internal.segment.Segment;
-import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.JobContext;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
index 5df0b83..1800fc3 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
@@ -30,7 +30,7 @@ import org.apache.carbondata.core.keygenerator.KeyGenException;
import org.apache.carbondata.core.update.CarbonUpdateUtil;
import org.apache.carbondata.core.update.SegmentUpdateDetails;
import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager;
-import org.apache.carbondata.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
index cdfe7a4..7585032 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
@@ -27,13 +27,13 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.hadoop.CarbonInputFormat;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.filter.FilterExpressionProcessor;
-import org.apache.carbondata.scan.filter.resolver.FilterResolverIntf;
-import org.apache.carbondata.scan.model.CarbonQueryPlan;
-import org.apache.carbondata.scan.model.QueryDimension;
-import org.apache.carbondata.scan.model.QueryMeasure;
-import org.apache.carbondata.scan.model.QueryModel;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.filter.FilterExpressionProcessor;
+import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
+import org.apache.carbondata.core.scan.model.CarbonQueryPlan;
+import org.apache.carbondata.core.scan.model.QueryDimension;
+import org.apache.carbondata.core.scan.model.QueryMeasure;
+import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Job;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
index a24a303..9b95f0a 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
@@ -28,7 +28,7 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.TableInfo;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.reader.ThriftReader;
import org.apache.thrift.TBase;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonInputFormat_FT.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonInputFormat_FT.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonInputFormat_FT.java
index 158af7d..5383646 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonInputFormat_FT.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonInputFormat_FT.java
@@ -23,10 +23,10 @@ import java.util.List;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.hadoop.CarbonInputFormat;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.LiteralExpression;
-import org.apache.carbondata.scan.expression.conditional.EqualToExpression;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.LiteralExpression;
+import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression;
import junit.framework.TestCase;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonInputMapperTest.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonInputMapperTest.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonInputMapperTest.java
index 359d4a7..def162c 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonInputMapperTest.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonInputMapperTest.java
@@ -31,10 +31,10 @@ import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.hadoop.CarbonInputFormat;
import org.apache.carbondata.hadoop.CarbonProjection;
import org.apache.carbondata.hadoop.test.util.StoreCreator;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.LiteralExpression;
-import org.apache.carbondata.scan.expression.conditional.EqualToExpression;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.LiteralExpression;
+import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression;
import junit.framework.TestCase;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/ObjectSerializationUtilTest.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/ObjectSerializationUtilTest.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/ObjectSerializationUtilTest.java
index e52d8d2..3b5c56b 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/ObjectSerializationUtilTest.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/ObjectSerializationUtilTest.java
@@ -21,10 +21,10 @@ package org.apache.carbondata.hadoop.test.util;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.hadoop.util.ObjectSerializationUtil;
-import org.apache.carbondata.scan.expression.ColumnExpression;
-import org.apache.carbondata.scan.expression.Expression;
-import org.apache.carbondata.scan.expression.LiteralExpression;
-import org.apache.carbondata.scan.expression.conditional.EqualToExpression;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.LiteralExpression;
+import org.apache.carbondata.core.scan.expression.conditional.EqualToExpression;
import junit.framework.TestCase;
import org.junit.Assert;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
index f4c0d72..fa3710f 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
@@ -47,7 +47,7 @@ import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.load.BlockDetails;
import org.apache.carbondata.core.load.LoadMetadataDetails;
import org.apache.carbondata.core.util.CarbonProperties;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
index f25e723..3555477 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
@@ -27,8 +27,8 @@ import org.apache.spark.sql.Row
import org.apache.spark.sql.common.util.QueryTest
import org.scalatest.BeforeAndAfterAll
+import org.apache.carbondata.core.datastorage.impl.FileFactory
import org.apache.carbondata.core.path.{CarbonStorePath, CarbonTablePath}
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory
class TestLoadDataGeneral extends QueryTest with BeforeAndAfterAll {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/blockprune/BlockPruneQueryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/blockprune/BlockPruneQueryTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/blockprune/BlockPruneQueryTestCase.scala
index 3eccc70..18efde8 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/blockprune/BlockPruneQueryTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/blockprune/BlockPruneQueryTestCase.scala
@@ -24,7 +24,7 @@ import org.apache.spark.sql.Row
import org.apache.spark.sql.common.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory
+import org.apache.carbondata.core.datastorage.impl.FileFactory
/**
* This class contains test cases for block prune query
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SubqueryWithFilterAndSortTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SubqueryWithFilterAndSortTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SubqueryWithFilterAndSortTestCase.scala
index 18cdf2a..9b4ac04 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SubqueryWithFilterAndSortTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/SubqueryWithFilterAndSortTestCase.scala
@@ -22,7 +22,7 @@ package org.apache.carbondata.spark.testsuite.detailquery
import org.apache.spark.sql.common.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory
+import org.apache.carbondata.core.datastorage.impl.FileFactory
import org.apache.carbondata.core.datastorage.store.impl.FileFactory.FileType
class SubqueryWithFilterAndSortTestCase extends QueryTest with BeforeAndAfterAll {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ValueCompressionDataTypeTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ValueCompressionDataTypeTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ValueCompressionDataTypeTestCase.scala
index 0baf230..81a2b00 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ValueCompressionDataTypeTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ValueCompressionDataTypeTestCase.scala
@@ -21,7 +21,7 @@ package org.apache.carbondata.spark.testsuite.detailquery
import org.apache.spark.sql.common.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory
+import org.apache.carbondata.core.datastorage.impl.FileFactory
import org.apache.carbondata.core.datastorage.store.impl.FileFactory.FileType
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
index ddbef8d..fbcccda 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
@@ -42,10 +42,10 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory.FileType;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.impl.FileFactory.FileType;
import org.apache.carbondata.core.load.LoadMetadataDetails;
import org.apache.carbondata.core.update.CarbonUpdateUtil;
import org.apache.carbondata.core.updatestatus.SegmentStatusManager;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java
index 62afa0f..16dfcba 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java
@@ -39,9 +39,9 @@ import org.apache.carbondata.core.CarbonTableIdentifier;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.load.LoadMetadataDetails;
import org.apache.carbondata.core.update.CarbonUpdateUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionExecutor.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionExecutor.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionExecutor.java
index e932eaa..1314499 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionExecutor.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionExecutor.java
@@ -38,14 +38,14 @@ import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.scan.executor.QueryExecutor;
-import org.apache.carbondata.scan.executor.QueryExecutorFactory;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
-import org.apache.carbondata.scan.model.QueryDimension;
-import org.apache.carbondata.scan.model.QueryMeasure;
-import org.apache.carbondata.scan.model.QueryModel;
-import org.apache.carbondata.scan.result.BatchResult;
-import org.apache.carbondata.scan.result.iterator.RawResultIterator;
+import org.apache.carbondata.core.scan.executor.QueryExecutor;
+import org.apache.carbondata.core.scan.executor.QueryExecutorFactory;
+import org.apache.carbondata.core.scan.executor.exception.QueryExecutionException;
+import org.apache.carbondata.core.scan.model.QueryDimension;
+import org.apache.carbondata.core.scan.model.QueryMeasure;
+import org.apache.carbondata.core.scan.model.QueryModel;
+import org.apache.carbondata.core.scan.result.BatchResult;
+import org.apache.carbondata.core.scan.result.iterator.RawResultIterator;
/**
* Executor class for executing the query on the selected segments to be merged.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionUtil.java
index 1cb2978..e9bd4ae 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionUtil.java
@@ -33,7 +33,7 @@ import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.util.CarbonUtil;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
index ea38d1a..5523dcf 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
@@ -33,9 +33,9 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.load.LoadMetadataDetails;
import org.apache.carbondata.core.reader.CarbonDeleteFilesDataReader;
import org.apache.carbondata.core.update.CarbonUpdateUtil;
[46/57] [abbrv] incubator-carbondata git commit: fix style
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/LocalCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/LocalCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/LocalCarbonFileTest.java
deleted file mode 100644
index 9d1ab42..0000000
--- a/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/LocalCarbonFileTest.java
+++ /dev/null
@@ -1,471 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.filesystem;
-
-import mockit.Mock;
-import mockit.MockUp;
-
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
-
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import sun.nio.ch.FileChannelImpl;
-
-import java.io.*;
-import java.nio.channels.ReadableByteChannel;
-import java.util.Objects;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-
-public class LocalCarbonFileTest {
-
- private static LocalCarbonFile localCarbonFile;
- private static File file;
- private static File dir;
- private static FileOutputStream oFile;
-
- @BeforeClass
- static public void setUp() {
- file = new File("Test.carbondata");
- dir = new File("Testdir.carbondata");
- if (!file.exists())
- try {
- file.createNewFile();
- dir.mkdir();
- } catch (IOException e) {
- e.printStackTrace();
- }
- try {
- oFile = new FileOutputStream(file, true);
-
-
- byte[] bytes = "core java api".getBytes();
-
- oFile.write(bytes);
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- localCarbonFile = new LocalCarbonFile(file);
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- @AfterClass
- static public void cleanUp() {
- file.delete();
- dir.delete();
-
- }
-
- @Test
- public void testListFilesWithCarbonFileFilterAndWithOutOutDirectoryPermission() {
- CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
- @Override
- public boolean accept(CarbonFile file) {
- return false;
- }
- };
- new MockUp<File>() {
- @Mock
- public boolean isDirectory() {
- return false;
- }
-
-
- };
- assertTrue(localCarbonFile.listFiles(carbonFileFilter) == null);
- }
-
- @Test
- public void testListFilesWithOutDirPermission() {
- localCarbonFile = new LocalCarbonFile(file);
- new MockUp<File>() {
- @Mock
- public boolean isDirectory() {
- return false;
- }
- };
- assertTrue(localCarbonFile.listFiles() == null);
- }
-
- @Test
- public void testCreateNewFileForException() throws IOException {
- localCarbonFile = new LocalCarbonFile(new File(""));
- assertTrue(!localCarbonFile.createNewFile());
- }
-
- @Test
- public void testCheckIfFileExists() throws IOException {
- localCarbonFile = new LocalCarbonFile(new File(""));
- assertTrue(!localCarbonFile.exists());
- }
-
- @Test
- public void testRenameForce() {
- localCarbonFile = new LocalCarbonFile(file);
- assertTrue(localCarbonFile.renameForce("Testdb.carbon"));
- File file1 = new File("Testdb.carbon");
- if (file1.exists()) {
- file1.delete();
- }
- }
-
- @Test
- public void testRenameTo() {
- localCarbonFile = new LocalCarbonFile(file);
- assertTrue(!localCarbonFile.renameTo("Testdb.carbon"));
- }
-
- @Test
- public void testsetLastModifiedTime() {
- localCarbonFile = new LocalCarbonFile(file);
- assertTrue(!localCarbonFile.setLastModifiedTime(50L));
- }
-
- @Test
- public void testtruncate() {
- localCarbonFile = new LocalCarbonFile(file);
- final int[] counter = {0};
- new MockUp<FileFactory>() {
- @Mock
- public FileFactory.FileType getFileType(String path) {
- {
- return FileFactory.FileType.LOCAL;
- }
- }
- };
- new MockUp<FileFactory>() {
- @Mock
- public boolean isFileExist(String filePath, FileFactory.FileType fileType) throws IOException {
- {
- return true;
- }
- }
- };
- new MockUp<CarbonFile>() {
- @Mock
- boolean delete() {
- return true;
- }
- };
- new MockUp<FileFactory>() {
- @Mock
- public boolean createNewFile(String filePath, FileFactory.FileType fileType) throws IOException {
- {
- return true;
- }
- }
- };
- new MockUp<FileFactory>() {
- @Mock
- public CarbonFile getCarbonFile(String path, FileFactory.FileType fileType) {
- {
- return new LocalCarbonFile(path);
- }
- }
- };
- new MockUp<CarbonFile>() {
- @Mock
- boolean delete() {
- return true;
- }
- };
-
- new MockUp<FileChannelImpl>() {
- @Mock
- public long transferFrom(ReadableByteChannel var1, long var2, long var4) throws IOException {
- if (counter[0] == 0) {
- counter[0] = counter[0] + 1;
- return 0L;
- } else {
- return 1L;
- }
- }
- };
- new MockUp<CarbonFile>() {
- @Mock
- boolean renameForce(String changetoName) {
- return true;
- }
- };
- localCarbonFile = new LocalCarbonFile(file);
- assertTrue(localCarbonFile.truncate(file.getName(), 1L));
- }
-
- @Test
- public void testtruncateForException() throws IOException {
- localCarbonFile = new LocalCarbonFile(file);
- new MockUp<FileFactory>() {
- @Mock
- public FileFactory.FileType getFileType(String path) {
- {
- return FileFactory.FileType.LOCAL;
- }
- }
- };
- new MockUp<FileFactory>() {
- @Mock
- public boolean isFileExist(String filePath, FileFactory.FileType fileType) throws IOException {
- {
- return true;
- }
- }
- };
- new MockUp<FileFactory>() {
- @Mock
- public CarbonFile getCarbonFile(String path, FileFactory.FileType fileType) {
- {
- return new LocalCarbonFile(path);
- }
- }
- };
- new MockUp<CarbonFile>() {
- @Mock
- boolean delete() {
- return true;
- }
- };
- new MockUp<FileFactory>() {
- @Mock
- public boolean createNewFile(String filePath, FileFactory.FileType fileType) throws IOException {
- {
- throw new IOException();
- }
- }
- };
-
-
- localCarbonFile.truncate(file.getName(), 2L);
- }
-
- @Test
- public void testListFilesWithDirPermission() {
- localCarbonFile = new LocalCarbonFile(file);
- new MockUp<File>() {
- @Mock
- public boolean isDirectory() {
- return true;
- }
- };
- new MockUp<File>() {
- @Mock
- public File[] listFiles() {
- return null;
- }
-
-
- };
- localCarbonFile = new LocalCarbonFile(dir);
- assertTrue(localCarbonFile.listFiles().length == 0);
- }
-
- @Test
- public void testListFilesWithCarbonFileFilterAndDirectoryPermission() {
- CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
- @Override
- public boolean accept(CarbonFile file) {
- return true;
- }
- };
- new MockUp<File>() {
- @Mock
- public boolean isDirectory() {
- return true;
- }
- };
- new MockUp<File>() {
- @Mock
- public File[] listFiles(FileFilter filter) {
-
- File[] file = new File[]{dir};
- return file;
- }
-
-
- };
-
- localCarbonFile = new LocalCarbonFile(dir);
-
- assertTrue(localCarbonFile.listFiles(carbonFileFilter).length == 1);
- }
-
- @Test
- public void testListFilesForNullWithCarbonFileFilterAndDirectoryPermission() {
- CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
- @Override
- public boolean accept(CarbonFile file) {
- return true;
- }
- };
- new MockUp<File>() {
- @Mock
- public File[] listFiles(FileFilter filter) {
- return null;
- }
-
-
- };
- new MockUp<File>() {
- @Mock
- public boolean isDirectory() {
- return false;
- }
-
- @Mock
- public File[] listFiles(FileFilter filter) {
- return null;
- }
-
-
- };
- localCarbonFile = new LocalCarbonFile(dir);
-
- assertTrue(localCarbonFile.listFiles(carbonFileFilter) == null);
- }
-
- @Test
- public void testListFilesForEmptyFileArrayWithCarbonFileFilter() {
- CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
- @Override
- public boolean accept(CarbonFile file) {
- return true;
- }
- };
- new MockUp<CarbonFileFilter>() {
- @Mock
- boolean accept(CarbonFile file) {
- return true;
- }
- };
- new MockUp<File>() {
- @Mock
- public File[] listFiles(FileFilter filter) {
- return null;
- }
- };
- new MockUp<File>() {
- @Mock
- public boolean isDirectory() {
- return true;
- }
-
- @Mock
- public File[] listFiles(FileFilter filter) {
- return null;
- }
- };
- localCarbonFile = new LocalCarbonFile(dir);
-
- assertTrue(localCarbonFile.listFiles(carbonFileFilter).length == 0);
- }
-
- @Test
- public void testFilesForConnicalPath() {
-
- new MockUp<File>() {
- @Mock
- public String getCanonicalPath() throws IOException {
- return "testFile";
- }
-
-
- };
- localCarbonFile = new LocalCarbonFile(dir);
-
- assertTrue(Objects.equals(localCarbonFile.getCanonicalPath(), "testFile"));
- }
-
- @Test
- public void testFilesForConnicalPathException() throws IOException {
-
- new MockUp<File>() {
- @Mock
- public String getCanonicalPath() throws IOException {
- throw new IOException();
- }
-
-
- };
- localCarbonFile = new LocalCarbonFile(dir);
-
- localCarbonFile.getCanonicalPath();
- }
-
- @Test
- public void testFilesForAbsolutePath() {
-
- new MockUp<File>() {
- @Mock
- public String getAbsolutePath() {
- return "testFile";
- }
-
-
- };
- localCarbonFile = new LocalCarbonFile(dir);
-
- assertEquals(localCarbonFile.getAbsolutePath(), "testFile");
- }
-
- @Test
- public void testFilesForGetPath() {
-
- new MockUp<File>() {
- @Mock
- public String getPath() {
- return "testFile";
- }
-
-
- };
- localCarbonFile = new LocalCarbonFile(dir);
-
- assertEquals(localCarbonFile.getPath(), "testFile");
- }
-
- @Test
- public void testFilesForFileExists() {
-
- localCarbonFile = new LocalCarbonFile(new File(""));
- assertEquals(localCarbonFile.exists(), false);
- }
-
- @Test
- public void testRenameForceForFileNotExists() {
- new MockUp<File>() {
- @Mock
- public boolean exists() {
- return false;
- }
-
- @Mock
- public boolean renameTo(File dest) {
- return true;
- }
- };
-
- localCarbonFile = new LocalCarbonFile("demo.txt");
-
- assertEquals(localCarbonFile.renameForce("Test.carbondata"), true);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/ViewFsCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/ViewFsCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/ViewFsCarbonFileTest.java
deleted file mode 100644
index 9cdef3a..0000000
--- a/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/ViewFsCarbonFileTest.java
+++ /dev/null
@@ -1,311 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.filesystem;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.viewfs.ViewFileSystem;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-
-public class ViewFsCarbonFileTest {
-
- private static ViewFSCarbonFile viewFSCarbonFile;
- private static FileStatus fileStatus;
- private static FileStatus fileStatusWithOutDirectoryPermission;
- private static String fileName;
- private static File file;
-
-
- @BeforeClass
- static public void setUp() {
- file = new File("Test.carbondata");
- if (!file.exists())
- try {
- file.createNewFile();
- } catch (IOException e) {
- e.printStackTrace();
- }
- try {
- FileOutputStream oFile = new FileOutputStream(file, true);
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- }
-
- fileStatus = new FileStatus(12L, true, 60, 120l, 180L, new Path(file.getAbsolutePath()));
- fileStatusWithOutDirectoryPermission = new FileStatus(12L, false, 60, 120l, 180L, new Path(file.getAbsolutePath()));
- fileName = file.getAbsolutePath();
- viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
- }
-
- @AfterClass
- static public void cleanUp() {
- file.delete();
- }
-
- @Test
- public void testRenameForceForException() throws IOException {
-
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- throw new IOException();
- }
-
- };
- viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
- viewFSCarbonFile.renameForce(fileName);
- }
-
- @Test
- public void testListFilesWithOutDirectoryPermission() {
- viewFSCarbonFile = new ViewFSCarbonFile(fileStatusWithOutDirectoryPermission);
- assertTrue(viewFSCarbonFile.listFiles() == null);
- }
-
- @Test
- public void testConstructorWithFilePath() {
- viewFSCarbonFile = new ViewFSCarbonFile(file.getAbsolutePath());
- assertTrue(viewFSCarbonFile instanceof ViewFSCarbonFile);
- }
-
- @Test
- public void testListFilesForNullListStatus() {
- viewFSCarbonFile = new ViewFSCarbonFile(fileStatusWithOutDirectoryPermission);
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new ViewFileSystem();
- }
-
- };
- new MockUp<ViewFileSystem>() {
- @Mock
- public FileStatus[] listStatus(Path var1) throws IOException {
-
- return null;
- }
-
- };
- new MockUp<ViewFileSystem>() {
- @Mock
- public boolean delete(Path var1, boolean var2) throws IOException {
-
- return true;
- }
-
- };
- //public boolean delete(Path var1, boolean var2) throws IOException;
- viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
- assertTrue(viewFSCarbonFile.listFiles().length == 0);
- }
-
- @Test
- public void testListDirectory() {
- viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new ViewFileSystem();
- }
-
- };
- new MockUp<ViewFileSystem>() {
- @Mock
- public FileStatus[] listStatus(Path var1) throws IOException {
-
- FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
- }
-
- };
-
- assertTrue(viewFSCarbonFile.listFiles().length == 1);
- }
-
- @Test
- public void testListFilesForException() throws IOException {
- viewFSCarbonFile = new ViewFSCarbonFile(fileStatusWithOutDirectoryPermission);
-
- new MockUp<FileStatus>() {
- @Mock
- public Path getPath() {
- return new Path(file.getAbsolutePath());
- }
-
- };
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- throw new IOException();
- }
-
- };
- new MockUp<ViewFileSystem>() {
- @Mock
- public FileStatus[] listStatus(Path var1) throws IOException {
-
- throw new IOException();
- }
-
- };
- viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
- viewFSCarbonFile.listFiles();
- }
-
- @Test
- public void testListFilesWithCarbonFilter() {
- CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
-
- @Override
- public boolean accept(CarbonFile file) {
- return true;
- }
- };
- viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
- assertTrue(viewFSCarbonFile.listFiles(carbonFileFilter).length == 1);
- }
-
- @Test
- public void testlistFilesWithoutFilter() {
- CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
-
- @Override
- public boolean accept(CarbonFile file) {
- return false;
- }
- };
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new ViewFileSystem();
- }
-
- };
- new MockUp<ViewFileSystem>() {
- @Mock
- public FileStatus[] listStatus(Path var1) throws IOException {
-
- FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
- }
-
- };
- viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
- assertTrue(viewFSCarbonFile.listFiles(carbonFileFilter).length == 0);
- }
-
- @Test
- public void testGetParentFIle() {
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
- new MockUp<Path>() {
- @Mock
- public Path getParent() {
- return new Path(file.getAbsolutePath()
- );
- }
-
- };
- new MockUp<FileStatus>() {
- @Mock
- public Path getPath() {
- return new Path(file.getAbsolutePath());
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public FileStatus getFileStatus(Path f) throws IOException {
-
- return new FileStatus(12L, true, 60, 120l, 180L, new Path(file.getAbsolutePath()));
- }
-
- };
-
- viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
- assertFalse(viewFSCarbonFile.getParentFile().equals(null));
- }
-
- @Test
- public void testForNonDisributedSystem() {
- viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new WebHdfsFileSystem();
- }
-
- };
- assertFalse(viewFSCarbonFile.renameForce(fileName));
- }
-
- @Test
- public void testrenameForceForViewFileSystem() {
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new ViewFileSystem();
- }
-
- };
- new MockUp<ViewFileSystem>() {
- @Mock
- public boolean delete(Path f, boolean recursive) throws
- IOException {
- return true;
-
- }
-
- };
- new MockUp<ViewFileSystem>() {
- @Mock
- public boolean rename(Path src, Path dst) throws IOException {
- return true;
-
- }
-
- };
-
- assertTrue(viewFSCarbonFile.renameForce(fileName));
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReaderTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReaderTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReaderTest.java
index 29fe16d..e723ac7 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReaderTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReaderTest.java
@@ -5,11 +5,11 @@ import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.List;
-import org.apache.carbondata.core.datastorage.FileHolder;
-import org.apache.carbondata.core.datastorage.compression.MeasureMetaDataModel;
-import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder;
-import org.apache.carbondata.core.datastorage.impl.data.compressed
+import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.compression.MeasureMetaDataModel;
+import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder;
+import org.apache.carbondata.core.datastore.impl.data.compressed
.HeavyCompressedDoubleArrayDataInMemoryStore;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.reader.measure
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreDataHolderUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreDataHolderUnitTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreDataHolderUnitTest.java
new file mode 100644
index 0000000..132b3a9
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/columnar/ColumnarKeyStoreDataHolderUnitTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.datastore.columnar;
+
+import java.util.List;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class ColumnarKeyStoreDataHolderUnitTest {
+
+ private static ColumnarKeyStoreDataHolder columnarKeyStoreDataHolder;
+ private static ColumnarKeyStoreMetadata columnarKeyStoreMetadata;
+
+ @BeforeClass public static void setup() {
+ byte[] keyBlockData = new byte[] { 16, 8, 32, 40, 8, 8, 8 };
+ int eachRowSize = 2;
+ int[] reverseIndex = new int[] { 1, 5, 6, 3, 8 };
+ columnarKeyStoreMetadata = new ColumnarKeyStoreMetadata(eachRowSize);
+ columnarKeyStoreMetadata.setColumnReverseIndex(reverseIndex);
+ columnarKeyStoreDataHolder =
+ new ColumnarKeyStoreDataHolder(keyBlockData, columnarKeyStoreMetadata);
+ }
+
+ @Test public void testGetSurrogateKeyWithNullINGetColumnReverseIndex() {
+ byte[] keyBlockData = new byte[] { 16, 8, 32, 40, 8, 8, 8 };
+ int eachRowSize = 1;
+ ColumnarKeyStoreMetadata columnarKeyStoreMetadata = new ColumnarKeyStoreMetadata(eachRowSize);
+ ColumnarKeyStoreDataHolder columnarKeyStoreDataHolderNew =
+ new ColumnarKeyStoreDataHolder(keyBlockData, columnarKeyStoreMetadata);
+ int columnIndex = 5;
+ int expected_result = 8;
+ int result = columnarKeyStoreDataHolderNew.getSurrogateKey(columnIndex);
+ assertEquals(expected_result, result);
+ }
+
+ @Test public void testGetSurrogateKeyWithNullINGetColumnReverseIndexAndRowSizeTwo() {
+ byte[] keyBlockData = new byte[] { 16, 8, 32, 40, 8, 8, 8 };
+ int eachRowSize = 2;
+ ColumnarKeyStoreMetadata columnarKeyStoreMetadata = new ColumnarKeyStoreMetadata(eachRowSize);
+ ColumnarKeyStoreDataHolder columnarKeyStoreDataHolderNew =
+ new ColumnarKeyStoreDataHolder(keyBlockData, columnarKeyStoreMetadata);
+ int columnIndex = 0;
+ int expected_result = 4104;
+ int result = columnarKeyStoreDataHolderNew.getSurrogateKey(columnIndex);
+ assertEquals(expected_result, result);
+ }
+
+ @Test public void testGetSurrogateKeyWithNotNullINGetColumnReverseIndex() {
+ int columnIndex = 0;
+ int expected_result = 8232;
+ int result = columnarKeyStoreDataHolder.getSurrogateKey(columnIndex);
+ assertEquals(expected_result, result);
+ }
+
+ @Test(expected = ArrayIndexOutOfBoundsException.class)
+ public void testExceptionInGetSurrogateKey() {
+ int columnIndex = 10;
+ int expected_result = 8232;
+ int result = columnarKeyStoreDataHolder.getSurrogateKey(columnIndex);
+ assertEquals(expected_result, result);
+ }
+
+ @Test public void testGetSurrogateKeyWithListOfByteWhileCreatingObject() {
+ byte[] keyBlockData = new byte[] { 32, 64, 32, 40, 64, 8, 8 };
+ List<byte[]> noDictionaryValBasedKeyBlockData = new java.util.ArrayList<>();
+ noDictionaryValBasedKeyBlockData.add(keyBlockData);
+ new ColumnarKeyStoreDataHolder(columnarKeyStoreMetadata);
+ int columnIndex = 0;
+ int expected_result = 8232;
+ int result = columnarKeyStoreDataHolder.getSurrogateKey(columnIndex);
+ assertEquals(expected_result, result);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java
new file mode 100644
index 0000000..aada6f8
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/AlluxioCarbonFileTest.java
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.filesystem;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Options;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class AlluxioCarbonFileTest {
+
+ private static AlluxioCarbonFile alluxioCarbonFile;
+ private static FileStatus fileStatus;
+ private static FileStatus fileStatusWithOutDirectoryPermission;
+ private static String fileName;
+ private static File file;
+
+
+ @BeforeClass
+ static public void setUp() {
+ file = new File("Test.carbondata");
+ if (!file.exists())
+ try {
+ file.createNewFile();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
+ FileOutputStream oFile = new FileOutputStream(file, true);
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
+
+ fileStatus = new FileStatus(12L, true, 60, 120l, 180L, new Path(file.getAbsolutePath()));
+ fileStatusWithOutDirectoryPermission = new FileStatus(12L, false, 60, 120l, 180L, new Path(file.getAbsolutePath()));
+ fileName = file.getAbsolutePath();
+ alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
+ }
+
+ @AfterClass
+ static public void cleanUp() {
+ file.delete();
+ }
+
+ @Test
+ public void testRenameForceForException() throws IOException {
+
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ throw new IOException();
+ }
+
+ };
+ alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
+ alluxioCarbonFile.renameForce(fileName);
+ }
+
+ @Test
+ public void testListFilesWithOutDirectoryPermission() {
+ alluxioCarbonFile = new AlluxioCarbonFile(fileStatusWithOutDirectoryPermission);
+ assertTrue(alluxioCarbonFile.listFiles() == null);
+ }
+
+ @Test
+ public void testConstructorWithFilePath() {
+ alluxioCarbonFile = new AlluxioCarbonFile(file.getAbsolutePath());
+ assertTrue(alluxioCarbonFile instanceof AlluxioCarbonFile);
+ }
+
+ @Test
+ public void testListFilesForNullListStatus() {
+ alluxioCarbonFile = new AlluxioCarbonFile(fileStatusWithOutDirectoryPermission);
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public FileStatus[] listStatus(Path var1) throws IOException {
+
+ return null;
+ }
+
+ };
+ alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
+ assertTrue(alluxioCarbonFile.listFiles().length == 0);
+ }
+
+ @Test
+ public void testListDirectory() {
+ alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public FileStatus[] listStatus(Path var1) throws IOException {
+
+ FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
+ return fileStatus;
+ }
+
+ };
+
+ assertTrue(alluxioCarbonFile.listFiles().length == 1);
+ }
+
+ @Test
+ public void testListFilesForException() throws IOException {
+ alluxioCarbonFile = new AlluxioCarbonFile(fileStatusWithOutDirectoryPermission);
+
+ new MockUp<FileStatus>() {
+ @Mock
+ public Path getPath() {
+ return new Path(file.getAbsolutePath());
+ }
+
+ };
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ throw new IOException();
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public FileStatus[] listStatus(Path var1) throws IOException {
+
+ throw new IOException();
+ }
+
+ };
+ alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
+ alluxioCarbonFile.listFiles();
+ }
+
+ @Test
+ public void testListFilesWithCarbonFilter() {
+ CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
+
+ @Override
+ public boolean accept(CarbonFile file) {
+ return true;
+ }
+ };
+ alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
+ assertTrue(alluxioCarbonFile.listFiles(carbonFileFilter).length == 1);
+ }
+
+ @Test
+ public void testlistFilesWithoutFilter() {
+ CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
+
+ @Override
+ public boolean accept(CarbonFile file) {
+ return false;
+ }
+ };
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public FileStatus[] listStatus(Path var1) throws IOException {
+
+ FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
+ return fileStatus;
+ }
+
+ };
+ alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
+ assertTrue(alluxioCarbonFile.listFiles(carbonFileFilter).length == 0);
+ }
+
+ @Test
+ public void testGetParentFile() {
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+ new MockUp<Path>() {
+ @Mock
+ public Path getParent() {
+ return new Path(file.getAbsolutePath()
+ );
+ }
+
+ };
+ new MockUp<FileStatus>() {
+ @Mock
+ public Path getPath() {
+ return new Path(file.getAbsolutePath());
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public FileStatus getFileStatus(Path path) throws IOException {
+
+ return new FileStatus(12L, true, 60, 120l, 180L, new Path(file.getAbsolutePath()));
+ }
+
+ };
+
+ alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
+ assertFalse(alluxioCarbonFile.getParentFile().equals(null));
+ }
+
+ @Test
+ public void testForNonDisributedSystem() {
+ alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new WebHdfsFileSystem();
+ }
+
+ };
+ assertFalse(alluxioCarbonFile.renameForce(fileName));
+ }
+
+ @Test
+ public void testrenameForceForDisributedSystem() {
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public void rename(Path src, Path dst, final Options.Rename... options) throws IOException {
+
+ }
+
+ };
+
+ alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
+ assertTrue(alluxioCarbonFile.renameForce(fileName));
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
new file mode 100644
index 0000000..a85280a
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/HDFSCarbonFileTest.java
@@ -0,0 +1,400 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.filesystem;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Options;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
+import org.apache.hadoop.util.Progressable;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+
+public class HDFSCarbonFileTest {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(HDFSCarbonFile.class.getName());
+ private static HDFSCarbonFile hdfsCarbonFile;
+ private static FileStatus fileStatus = null;
+ private static FileStatus fileStatusWithOutDirectoryPermission;
+ private static String fileName = null;
+ private static FileSystem fs = null;
+ private static Path pt;
+
+ @BeforeClass
+ static public void setUp() throws IOException {
+ Configuration config = new Configuration();
+//adding local hadoop configuration
+ config.addResource(new Path("core-site.xml"));
+ config.addResource(new Path("hdfs-site.xml"));
+ fileName = "Test.carbondata"; //this path is HDFS path
+ pt = new Path(fileName);
+ fs = FileSystem.get(new Configuration(config));
+ fs.create(pt);
+ if (fs.exists(pt)) {
+ OutputStream os = fs.create(pt,
+ new Progressable() {
+ public void progress() {
+ LOGGER.info("Started Writing to File===");
+ }
+ });
+ BufferedWriter br = new BufferedWriter(new OutputStreamWriter(os, "UTF-8"));
+ br.write("Hello World");
+ br.close();
+ fs.close();
+
+ fileStatus = new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName));
+ fileStatusWithOutDirectoryPermission = new FileStatus(12L, false, 60, 120l, 180L, new Path(fileName));
+ hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
+
+ }
+ }
+
+ @AfterClass
+ static public void cleanUp() {
+ try {
+ fs.delete(pt, true);
+ } catch (IOException e) {
+ LOGGER.error("Exception Occured" + e.getMessage());
+ }
+ }
+
+ @Test
+ public void testRenameForceForException() throws IOException {
+
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ throw new IOException();
+ }
+
+ };
+ hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
+ hdfsCarbonFile.renameForce(fileName);
+ }
+
+ @Test
+ public void testListFilesWithOutDirectoryPermission() {
+ hdfsCarbonFile = new HDFSCarbonFile(fileStatusWithOutDirectoryPermission);
+ new MockUp<FileStatus>() {
+ @Mock
+ public boolean isDirectory() {
+ return false;
+ }
+
+ };
+
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+
+ new MockUp<FileStatus>() {
+ @Mock
+ public Path getPath() {
+ return new Path(fileName);
+ }
+
+ };
+ assertEquals(hdfsCarbonFile.listFiles(), null);
+ }
+
+ @Test
+ public void testConstructorWithFilePath() {
+ hdfsCarbonFile = new HDFSCarbonFile(fileName);
+ assertTrue(hdfsCarbonFile instanceof HDFSCarbonFile);
+ }
+
+ @Test
+ public void testListFilesForNullListStatus() {
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public FileStatus[] listStatus(Path var1) throws IOException {
+
+ return null;
+ }
+
+ };
+ hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
+ assertEquals(hdfsCarbonFile.listFiles().length, 0);
+ }
+
+ @Test
+ public void testListDirectory() {
+ hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public FileStatus[] listStatus(Path var1) throws IOException {
+
+ FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
+ return fileStatus;
+ }
+
+ };
+
+ assertEquals(hdfsCarbonFile.listFiles().length, 1);
+ }
+
+ @Test
+ public void testListFilesForException() throws IOException {
+ new HDFSCarbonFile(fileStatusWithOutDirectoryPermission);
+
+ new MockUp<FileStatus>() {
+ @Mock
+ public Path getPath() {
+ return new Path(fileName);
+ }
+
+ };
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ throw new IOException();
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public FileStatus[] listStatus(Path var1) throws IOException {
+
+ throw new IOException();
+ }
+
+ };
+ hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
+ hdfsCarbonFile.listFiles();
+ }
+
+ @Test
+ public void testListFilesWithCarbonFilter() {
+ CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
+
+ @Override
+ public boolean accept(CarbonFile file) {
+ return true;
+ }
+ };
+ new MockUp<FileStatus>() {
+ @Mock
+ public boolean isDirectory() {
+ return true;
+ }
+
+ };
+
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+
+ new MockUp<FileStatus>() {
+ @Mock
+ public Path getPath() {
+ return new Path(fileName);
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public FileStatus[] listStatus(Path var1) throws IOException {
+
+ FileStatus fileStatus[] = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
+ return fileStatus;
+ }
+
+ };
+ hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
+ assertEquals(hdfsCarbonFile.listFiles(carbonFileFilter).length, 1);
+ }
+
+ @Test
+ public void testlistFilesWithoutFilter() {
+ CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
+
+ @Override
+ public boolean accept(CarbonFile file) {
+ return false;
+ }
+ };
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public FileStatus[] listStatus(Path var1) throws IOException {
+
+ FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
+ return fileStatus;
+ }
+
+ };
+ hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
+ assertEquals(hdfsCarbonFile.listFiles(carbonFileFilter).length, 0);
+ }
+
+ @Test
+ public void testGetParentFileForNull() {
+
+ new MockUp<Path>() {
+ @Mock
+ public Path getParent() {
+ return null;
+ }
+
+ };
+ new MockUp<FileStatus>() {
+ @Mock
+ public Path getPath() {
+ return new Path(fileName);
+ }
+
+ };
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+
+ new MockUp<FileStatus>() {
+ @Mock
+ public Path getPath() {
+ return new Path(fileName);
+ }
+
+ };
+ hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
+ assertEquals(hdfsCarbonFile.getParentFile(), null);
+ }
+
+ @Test
+ public void testGetParentFile() {
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+ new MockUp<Path>() {
+ @Mock
+ public Path getParent() {
+ return new Path(fileName);
+ }
+
+ };
+ new MockUp<FileStatus>() {
+ @Mock
+ public Path getPath() {
+ return new Path(fileName);
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public FileStatus getFileStatus(Path file) throws IOException {
+
+ return new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName));
+ }
+
+ };
+
+ hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
+ assertTrue(hdfsCarbonFile.getParentFile() instanceof CarbonFile);
+ }
+
+ @Test
+ public void testForNonDisributedSystem() {
+ new HDFSCarbonFile(fileStatus);
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new WebHdfsFileSystem();
+ }
+
+ };
+ assertEquals(hdfsCarbonFile.renameForce(fileName), false);
+ }
+
+ @Test
+ public void testrenameForceForDisributedSystem() {
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public void rename(Path src, Path dst, final Options.Rename... options) throws IOException {
+
+ }
+
+ };
+ hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
+ assertEquals(hdfsCarbonFile.renameForce(fileName), true);
+
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
new file mode 100644
index 0000000..4e2c650
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFileTest.java
@@ -0,0 +1,471 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.filesystem;
+
+import mockit.Mock;
+import mockit.MockUp;
+
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import sun.nio.ch.FileChannelImpl;
+
+import java.io.*;
+import java.nio.channels.ReadableByteChannel;
+import java.util.Objects;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+
+public class LocalCarbonFileTest {
+
+ private static LocalCarbonFile localCarbonFile;
+ private static File file;
+ private static File dir;
+ private static FileOutputStream oFile;
+
+ @BeforeClass
+ static public void setUp() {
+ file = new File("Test.carbondata");
+ dir = new File("Testdir.carbondata");
+ if (!file.exists())
+ try {
+ file.createNewFile();
+ dir.mkdir();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
+ oFile = new FileOutputStream(file, true);
+
+
+ byte[] bytes = "core java api".getBytes();
+
+ oFile.write(bytes);
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ localCarbonFile = new LocalCarbonFile(file);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ @AfterClass
+ static public void cleanUp() {
+ file.delete();
+ dir.delete();
+
+ }
+
+ @Test
+ public void testListFilesWithCarbonFileFilterAndWithOutOutDirectoryPermission() {
+ CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
+ @Override
+ public boolean accept(CarbonFile file) {
+ return false;
+ }
+ };
+ new MockUp<File>() {
+ @Mock
+ public boolean isDirectory() {
+ return false;
+ }
+
+
+ };
+ assertTrue(localCarbonFile.listFiles(carbonFileFilter) == null);
+ }
+
+ @Test
+ public void testListFilesWithOutDirPermission() {
+ localCarbonFile = new LocalCarbonFile(file);
+ new MockUp<File>() {
+ @Mock
+ public boolean isDirectory() {
+ return false;
+ }
+ };
+ assertTrue(localCarbonFile.listFiles() == null);
+ }
+
+ @Test
+ public void testCreateNewFileForException() throws IOException {
+ localCarbonFile = new LocalCarbonFile(new File(""));
+ assertTrue(!localCarbonFile.createNewFile());
+ }
+
+ @Test
+ public void testCheckIfFileExists() throws IOException {
+ localCarbonFile = new LocalCarbonFile(new File(""));
+ assertTrue(!localCarbonFile.exists());
+ }
+
+ @Test
+ public void testRenameForce() {
+ localCarbonFile = new LocalCarbonFile(file);
+ assertTrue(localCarbonFile.renameForce("Testdb.carbon"));
+ File file1 = new File("Testdb.carbon");
+ if (file1.exists()) {
+ file1.delete();
+ }
+ }
+
+ @Test
+ public void testRenameTo() {
+ localCarbonFile = new LocalCarbonFile(file);
+ assertTrue(!localCarbonFile.renameTo("Testdb.carbon"));
+ }
+
+ @Test
+ public void testsetLastModifiedTime() {
+ localCarbonFile = new LocalCarbonFile(file);
+ assertTrue(!localCarbonFile.setLastModifiedTime(50L));
+ }
+
+ @Test
+ public void testtruncate() {
+ localCarbonFile = new LocalCarbonFile(file);
+ final int[] counter = {0};
+ new MockUp<FileFactory>() {
+ @Mock
+ public FileFactory.FileType getFileType(String path) {
+ {
+ return FileFactory.FileType.LOCAL;
+ }
+ }
+ };
+ new MockUp<FileFactory>() {
+ @Mock
+ public boolean isFileExist(String filePath, FileFactory.FileType fileType) throws IOException {
+ {
+ return true;
+ }
+ }
+ };
+ new MockUp<CarbonFile>() {
+ @Mock
+ boolean delete() {
+ return true;
+ }
+ };
+ new MockUp<FileFactory>() {
+ @Mock
+ public boolean createNewFile(String filePath, FileFactory.FileType fileType) throws IOException {
+ {
+ return true;
+ }
+ }
+ };
+ new MockUp<FileFactory>() {
+ @Mock
+ public CarbonFile getCarbonFile(String path, FileFactory.FileType fileType) {
+ {
+ return new LocalCarbonFile(path);
+ }
+ }
+ };
+ new MockUp<CarbonFile>() {
+ @Mock
+ boolean delete() {
+ return true;
+ }
+ };
+
+ new MockUp<FileChannelImpl>() {
+ @Mock
+ public long transferFrom(ReadableByteChannel var1, long var2, long var4) throws IOException {
+ if (counter[0] == 0) {
+ counter[0] = counter[0] + 1;
+ return 0L;
+ } else {
+ return 1L;
+ }
+ }
+ };
+ new MockUp<CarbonFile>() {
+ @Mock
+ boolean renameForce(String changetoName) {
+ return true;
+ }
+ };
+ localCarbonFile = new LocalCarbonFile(file);
+ assertTrue(localCarbonFile.truncate(file.getName(), 1L));
+ }
+
+ @Test
+ public void testtruncateForException() throws IOException {
+ localCarbonFile = new LocalCarbonFile(file);
+ new MockUp<FileFactory>() {
+ @Mock
+ public FileFactory.FileType getFileType(String path) {
+ {
+ return FileFactory.FileType.LOCAL;
+ }
+ }
+ };
+ new MockUp<FileFactory>() {
+ @Mock
+ public boolean isFileExist(String filePath, FileFactory.FileType fileType) throws IOException {
+ {
+ return true;
+ }
+ }
+ };
+ new MockUp<FileFactory>() {
+ @Mock
+ public CarbonFile getCarbonFile(String path, FileFactory.FileType fileType) {
+ {
+ return new LocalCarbonFile(path);
+ }
+ }
+ };
+ new MockUp<CarbonFile>() {
+ @Mock
+ boolean delete() {
+ return true;
+ }
+ };
+ new MockUp<FileFactory>() {
+ @Mock
+ public boolean createNewFile(String filePath, FileFactory.FileType fileType) throws IOException {
+ {
+ throw new IOException();
+ }
+ }
+ };
+
+
+ localCarbonFile.truncate(file.getName(), 2L);
+ }
+
+ @Test
+ public void testListFilesWithDirPermission() {
+ localCarbonFile = new LocalCarbonFile(file);
+ new MockUp<File>() {
+ @Mock
+ public boolean isDirectory() {
+ return true;
+ }
+ };
+ new MockUp<File>() {
+ @Mock
+ public File[] listFiles() {
+ return null;
+ }
+
+
+ };
+ localCarbonFile = new LocalCarbonFile(dir);
+ assertTrue(localCarbonFile.listFiles().length == 0);
+ }
+
+ @Test
+ public void testListFilesWithCarbonFileFilterAndDirectoryPermission() {
+ CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
+ @Override
+ public boolean accept(CarbonFile file) {
+ return true;
+ }
+ };
+ new MockUp<File>() {
+ @Mock
+ public boolean isDirectory() {
+ return true;
+ }
+ };
+ new MockUp<File>() {
+ @Mock
+ public File[] listFiles(FileFilter filter) {
+
+ File[] file = new File[]{dir};
+ return file;
+ }
+
+
+ };
+
+ localCarbonFile = new LocalCarbonFile(dir);
+
+ assertTrue(localCarbonFile.listFiles(carbonFileFilter).length == 1);
+ }
+
+ @Test
+ public void testListFilesForNullWithCarbonFileFilterAndDirectoryPermission() {
+ CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
+ @Override
+ public boolean accept(CarbonFile file) {
+ return true;
+ }
+ };
+ new MockUp<File>() {
+ @Mock
+ public File[] listFiles(FileFilter filter) {
+ return null;
+ }
+
+
+ };
+ new MockUp<File>() {
+ @Mock
+ public boolean isDirectory() {
+ return false;
+ }
+
+ @Mock
+ public File[] listFiles(FileFilter filter) {
+ return null;
+ }
+
+
+ };
+ localCarbonFile = new LocalCarbonFile(dir);
+
+ assertTrue(localCarbonFile.listFiles(carbonFileFilter) == null);
+ }
+
+ @Test
+ public void testListFilesForEmptyFileArrayWithCarbonFileFilter() {
+ CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
+ @Override
+ public boolean accept(CarbonFile file) {
+ return true;
+ }
+ };
+ new MockUp<CarbonFileFilter>() {
+ @Mock
+ boolean accept(CarbonFile file) {
+ return true;
+ }
+ };
+ new MockUp<File>() {
+ @Mock
+ public File[] listFiles(FileFilter filter) {
+ return null;
+ }
+ };
+ new MockUp<File>() {
+ @Mock
+ public boolean isDirectory() {
+ return true;
+ }
+
+ @Mock
+ public File[] listFiles(FileFilter filter) {
+ return null;
+ }
+ };
+ localCarbonFile = new LocalCarbonFile(dir);
+
+ assertTrue(localCarbonFile.listFiles(carbonFileFilter).length == 0);
+ }
+
+ @Test
+ public void testFilesForConnicalPath() {
+
+ new MockUp<File>() {
+ @Mock
+ public String getCanonicalPath() throws IOException {
+ return "testFile";
+ }
+
+
+ };
+ localCarbonFile = new LocalCarbonFile(dir);
+
+ assertTrue(Objects.equals(localCarbonFile.getCanonicalPath(), "testFile"));
+ }
+
+ @Test
+ public void testFilesForConnicalPathException() throws IOException {
+
+ new MockUp<File>() {
+ @Mock
+ public String getCanonicalPath() throws IOException {
+ throw new IOException();
+ }
+
+
+ };
+ localCarbonFile = new LocalCarbonFile(dir);
+
+ localCarbonFile.getCanonicalPath();
+ }
+
+ @Test
+ public void testFilesForAbsolutePath() {
+
+ new MockUp<File>() {
+ @Mock
+ public String getAbsolutePath() {
+ return "testFile";
+ }
+
+
+ };
+ localCarbonFile = new LocalCarbonFile(dir);
+
+ assertEquals(localCarbonFile.getAbsolutePath(), "testFile");
+ }
+
+ @Test
+ public void testFilesForGetPath() {
+
+ new MockUp<File>() {
+ @Mock
+ public String getPath() {
+ return "testFile";
+ }
+
+
+ };
+ localCarbonFile = new LocalCarbonFile(dir);
+
+ assertEquals(localCarbonFile.getPath(), "testFile");
+ }
+
+ @Test
+ public void testFilesForFileExists() {
+
+ localCarbonFile = new LocalCarbonFile(new File(""));
+ assertEquals(localCarbonFile.exists(), false);
+ }
+
+ @Test
+ public void testRenameForceForFileNotExists() {
+ new MockUp<File>() {
+ @Mock
+ public boolean exists() {
+ return false;
+ }
+
+ @Mock
+ public boolean renameTo(File dest) {
+ return true;
+ }
+ };
+
+ localCarbonFile = new LocalCarbonFile("demo.txt");
+
+ assertEquals(localCarbonFile.renameForce("Test.carbondata"), true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java
new file mode 100644
index 0000000..7569dc6
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/filesystem/ViewFsCarbonFileTest.java
@@ -0,0 +1,311 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.filesystem;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.viewfs.ViewFileSystem;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+
+public class ViewFsCarbonFileTest {
+
+ private static ViewFSCarbonFile viewFSCarbonFile;
+ private static FileStatus fileStatus;
+ private static FileStatus fileStatusWithOutDirectoryPermission;
+ private static String fileName;
+ private static File file;
+
+
+ @BeforeClass
+ static public void setUp() {
+ file = new File("Test.carbondata");
+ if (!file.exists())
+ try {
+ file.createNewFile();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ try {
+ FileOutputStream oFile = new FileOutputStream(file, true);
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
+
+ fileStatus = new FileStatus(12L, true, 60, 120l, 180L, new Path(file.getAbsolutePath()));
+ fileStatusWithOutDirectoryPermission = new FileStatus(12L, false, 60, 120l, 180L, new Path(file.getAbsolutePath()));
+ fileName = file.getAbsolutePath();
+ viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
+ }
+
+ @AfterClass
+ static public void cleanUp() {
+ file.delete();
+ }
+
+ @Test
+ public void testRenameForceForException() throws IOException {
+
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ throw new IOException();
+ }
+
+ };
+ viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
+ viewFSCarbonFile.renameForce(fileName);
+ }
+
+ @Test
+ public void testListFilesWithOutDirectoryPermission() {
+ viewFSCarbonFile = new ViewFSCarbonFile(fileStatusWithOutDirectoryPermission);
+ assertTrue(viewFSCarbonFile.listFiles() == null);
+ }
+
+ @Test
+ public void testConstructorWithFilePath() {
+ viewFSCarbonFile = new ViewFSCarbonFile(file.getAbsolutePath());
+ assertTrue(viewFSCarbonFile instanceof ViewFSCarbonFile);
+ }
+
+ @Test
+ public void testListFilesForNullListStatus() {
+ viewFSCarbonFile = new ViewFSCarbonFile(fileStatusWithOutDirectoryPermission);
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new ViewFileSystem();
+ }
+
+ };
+ new MockUp<ViewFileSystem>() {
+ @Mock
+ public FileStatus[] listStatus(Path var1) throws IOException {
+
+ return null;
+ }
+
+ };
+ new MockUp<ViewFileSystem>() {
+ @Mock
+ public boolean delete(Path var1, boolean var2) throws IOException {
+
+ return true;
+ }
+
+ };
+ //public boolean delete(Path var1, boolean var2) throws IOException;
+ viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
+ assertTrue(viewFSCarbonFile.listFiles().length == 0);
+ }
+
+ @Test
+ public void testListDirectory() {
+ viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new ViewFileSystem();
+ }
+
+ };
+ new MockUp<ViewFileSystem>() {
+ @Mock
+ public FileStatus[] listStatus(Path var1) throws IOException {
+
+ FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
+ return fileStatus;
+ }
+
+ };
+
+ assertTrue(viewFSCarbonFile.listFiles().length == 1);
+ }
+
+ @Test
+ public void testListFilesForException() throws IOException {
+ viewFSCarbonFile = new ViewFSCarbonFile(fileStatusWithOutDirectoryPermission);
+
+ new MockUp<FileStatus>() {
+ @Mock
+ public Path getPath() {
+ return new Path(file.getAbsolutePath());
+ }
+
+ };
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ throw new IOException();
+ }
+
+ };
+ new MockUp<ViewFileSystem>() {
+ @Mock
+ public FileStatus[] listStatus(Path var1) throws IOException {
+
+ throw new IOException();
+ }
+
+ };
+ viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
+ viewFSCarbonFile.listFiles();
+ }
+
+ @Test
+ public void testListFilesWithCarbonFilter() {
+ CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
+
+ @Override
+ public boolean accept(CarbonFile file) {
+ return true;
+ }
+ };
+ viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
+ assertTrue(viewFSCarbonFile.listFiles(carbonFileFilter).length == 1);
+ }
+
+ @Test
+ public void testlistFilesWithoutFilter() {
+ CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
+
+ @Override
+ public boolean accept(CarbonFile file) {
+ return false;
+ }
+ };
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new ViewFileSystem();
+ }
+
+ };
+ new MockUp<ViewFileSystem>() {
+ @Mock
+ public FileStatus[] listStatus(Path var1) throws IOException {
+
+ FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
+ return fileStatus;
+ }
+
+ };
+ viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
+ assertTrue(viewFSCarbonFile.listFiles(carbonFileFilter).length == 0);
+ }
+
+ @Test
+ public void testGetParentFIle() {
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new DistributedFileSystem();
+ }
+
+ };
+ new MockUp<Path>() {
+ @Mock
+ public Path getParent() {
+ return new Path(file.getAbsolutePath()
+ );
+ }
+
+ };
+ new MockUp<FileStatus>() {
+ @Mock
+ public Path getPath() {
+ return new Path(file.getAbsolutePath());
+ }
+
+ };
+ new MockUp<DistributedFileSystem>() {
+ @Mock
+ public FileStatus getFileStatus(Path f) throws IOException {
+
+ return new FileStatus(12L, true, 60, 120l, 180L, new Path(file.getAbsolutePath()));
+ }
+
+ };
+
+ viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
+ assertFalse(viewFSCarbonFile.getParentFile().equals(null));
+ }
+
+ @Test
+ public void testForNonDisributedSystem() {
+ viewFSCarbonFile = new ViewFSCarbonFile(fileStatus);
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new WebHdfsFileSystem();
+ }
+
+ };
+ assertFalse(viewFSCarbonFile.renameForce(fileName));
+ }
+
+ @Test
+ public void testrenameForceForViewFileSystem() {
+ new MockUp<Path>() {
+ @Mock
+ public FileSystem getFileSystem(Configuration conf) throws IOException {
+ return new ViewFileSystem();
+ }
+
+ };
+ new MockUp<ViewFileSystem>() {
+ @Mock
+ public boolean delete(Path f, boolean recursive) throws
+ IOException {
+ return true;
+
+ }
+
+ };
+ new MockUp<ViewFileSystem>() {
+ @Mock
+ public boolean rename(Path src, Path dst) throws IOException {
+ return true;
+
+ }
+
+ };
+
+ assertTrue(viewFSCarbonFile.renameForce(fileName));
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java b/core/src/test/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java
index e0708dc..585f33b 100644
--- a/core/src/test/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java
@@ -26,8 +26,8 @@ import java.util.UUID;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnIdentifier;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.writer.CarbonDictionaryWriter;
import org.apache.carbondata.core.writer.CarbonDictionaryWriterImpl;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
index 5855262..7e9e18d 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
@@ -24,7 +24,7 @@ import mockit.MockUp;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.metadata.index.BlockIndexInfo;
-import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
import org.apache.carbondata.core.metadata.BlockletInfoColumnar;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.format.*;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
index 37f1ef6..c928d36 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
@@ -31,10 +31,10 @@ import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.datastorage.columnar.ColumnGroupModel;
-import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastorage.filesystem.LocalCarbonFile;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
+import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastore.filesystem.LocalCarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.core.scan.model.QueryDimension;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java b/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java
index f1b8070..9b90755 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java
@@ -26,9 +26,9 @@ import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.metadata.blocklet.SegmentInfo;
-import org.apache.carbondata.core.datastorage.FileHolder;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
-import org.apache.carbondata.core.datastorage.impl.FileHolderImpl;
+import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.datastore.impl.FileHolderImpl;
import org.apache.carbondata.core.reader.CarbonFooterReader;
import org.apache.carbondata.core.reader.CarbonIndexFileReader;
import org.apache.carbondata.core.reader.ThriftReader;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java
index 17fe306..0e024e9 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java
@@ -23,29 +23,29 @@ import static org.junit.Assert.assertTrue;
import java.nio.ByteBuffer;
-import org.apache.carbondata.core.datastorage.compression.MeasureMetaDataModel;
-import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastorage.compression.decimal.CompressionMaxMinByte;
-import org.apache.carbondata.core.datastorage.compression.decimal.CompressionMaxMinDefault;
-import org.apache.carbondata.core.datastorage.compression.decimal.CompressionMaxMinInt;
-import org.apache.carbondata.core.datastorage.compression.decimal.CompressionMaxMinLong;
-import org.apache.carbondata.core.datastorage.compression.decimal.CompressionMaxMinShort;
-import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalByte;
-import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalDefault;
-import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalInt;
-import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalLong;
-import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalMaxMinByte;
-import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalMaxMinDefault;
-import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalMaxMinInt;
-import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalMaxMinLong;
-import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalMaxMinShort;
-import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalShort;
-import org.apache.carbondata.core.datastorage.compression.none.CompressionNoneByte;
-import org.apache.carbondata.core.datastorage.compression.none.CompressionNoneDefault;
-import org.apache.carbondata.core.datastorage.compression.none.CompressionNoneInt;
-import org.apache.carbondata.core.datastorage.compression.none.CompressionNoneLong;
-import org.apache.carbondata.core.datastorage.compression.none.CompressionNoneShort;
+import org.apache.carbondata.core.datastore.compression.MeasureMetaDataModel;
+import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.compression.decimal.CompressionMaxMinByte;
+import org.apache.carbondata.core.datastore.compression.decimal.CompressionMaxMinDefault;
+import org.apache.carbondata.core.datastore.compression.decimal.CompressionMaxMinInt;
+import org.apache.carbondata.core.datastore.compression.decimal.CompressionMaxMinLong;
+import org.apache.carbondata.core.datastore.compression.decimal.CompressionMaxMinShort;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalByte;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalDefault;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalInt;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalLong;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalMaxMinByte;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalMaxMinDefault;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalMaxMinInt;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalMaxMinLong;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalMaxMinShort;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalShort;
+import org.apache.carbondata.core.datastore.compression.none.CompressionNoneByte;
+import org.apache.carbondata.core.datastore.compression.none.CompressionNoneDefault;
+import org.apache.carbondata.core.datastore.compression.none.CompressionNoneInt;
+import org.apache.carbondata.core.datastore.compression.none.CompressionNoneLong;
+import org.apache.carbondata.core.datastore.compression.none.CompressionNoneShort;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java b/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java
index 4994942..bee220b 100644
--- a/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java
@@ -39,8 +39,8 @@ import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk;
import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReaderImpl;
import org.apache.carbondata.core.reader.CarbonDictionaryReaderImpl;
[57/57] [abbrv] incubator-carbondata git commit: fix style
Posted by ja...@apache.org.
fix style
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/afa88ee6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/afa88ee6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/afa88ee6
Branch: refs/heads/movecorepackage
Commit: afa88ee654fc76fcf59c70d51d378a0e3fa75e71
Parents: f75ddfb
Author: jackylk <ja...@huawei.com>
Authored: Sat Jan 14 15:56:10 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Sat Jan 14 15:56:10 2017 +0800
----------------------------------------------------------------------
.../core/locks/AbstractCarbonLock.java | 77 ++++++++
.../core/locks/CarbonLockFactory.java | 94 +++++++++
.../carbondata/core/locks/CarbonLockUtil.java | 64 ++++++
.../carbondata/core/locks/HdfsFileLock.java | 129 ++++++++++++
.../carbondata/core/locks/ICarbonLock.java | 40 ++++
.../carbondata/core/locks/LocalFileLock.java | 164 ++++++++++++++++
.../apache/carbondata/core/locks/LockUsage.java | 36 ++++
.../carbondata/core/locks/ZooKeeperLocking.java | 195 +++++++++++++++++++
.../carbondata/core/locks/ZookeeperInit.java | 82 ++++++++
.../core/update/CarbonUpdateUtil.java | 2 +-
.../core/updatestatus/SegmentStatusManager.java | 8 +-
.../SegmentUpdateStatusManager.java | 6 +-
.../updatestatus/locks/AbstractCarbonLock.java | 77 --------
.../updatestatus/locks/CarbonLockFactory.java | 94 ---------
.../core/updatestatus/locks/CarbonLockUtil.java | 64 ------
.../core/updatestatus/locks/HdfsFileLock.java | 129 ------------
.../core/updatestatus/locks/ICarbonLock.java | 40 ----
.../core/updatestatus/locks/LocalFileLock.java | 164 ----------------
.../core/updatestatus/locks/LockUsage.java | 36 ----
.../updatestatus/locks/ZooKeeperLocking.java | 195 -------------------
.../core/updatestatus/locks/ZookeeperInit.java | 82 --------
.../core/locks/LocalFileLockTest.java | 64 ++++++
.../core/locks/ZooKeeperLockingTest.java | 142 ++++++++++++++
.../updatestatus/locks/LocalFileLockTest.java | 64 ------
.../locks/ZooKeeperLockingTest.java | 142 --------------
.../datacompaction/DataCompactionLockTest.scala | 2 +-
.../dataretention/DataRetentionTestCase.scala | 2 +-
.../carbondata/spark/load/CarbonLoaderUtil.java | 2 +-
.../spark/merger/CarbonDataMergerUtil.java | 6 +-
.../spark/rdd/CarbonGlobalDictionaryRDD.scala | 2 +-
.../spark/rdd/DataManagementFunc.scala | 2 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 2 +-
.../sql/execution/command/IUDCommands.scala | 2 +-
.../execution/command/carbonTableSchema.scala | 2 +-
.../apache/spark/sql/hive/CarbonMetastore.scala | 2 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 8 +-
.../spark/sql/CarbonDataFrameWriter.scala | 2 +-
.../scala/org/apache/spark/sql/CarbonScan.scala | 2 -
.../org/apache/spark/sql/TableCreator.scala | 2 +-
.../execution/CarbonLateDecodeStrategy.scala | 2 +-
.../execution/command/carbonTableSchema.scala | 4 +-
.../apache/spark/sql/hive/CarbonMetastore.scala | 8 +-
42 files changed, 1120 insertions(+), 1122 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/core/src/main/java/org/apache/carbondata/core/locks/AbstractCarbonLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/AbstractCarbonLock.java b/core/src/main/java/org/apache/carbondata/core/locks/AbstractCarbonLock.java
new file mode 100644
index 0000000..7ced442
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/locks/AbstractCarbonLock.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.locks;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+/**
+ * This is the abstract class of the lock implementations.This handles the
+ * retrying part of the locking.
+ */
+abstract class AbstractCarbonLock implements ICarbonLock {
+ private int retryCount;
+
+ private int retryTimeout;
+
+ public abstract boolean lock();
+
+ /**
+ * API for enabling the locking of file with retries.
+ */
+ public boolean lockWithRetries() {
+ try {
+ for (int i = 0; i < retryCount; i++) {
+ if (lock()) {
+ return true;
+ } else {
+ Thread.sleep(retryTimeout * 1000L);
+ }
+ }
+ } catch (InterruptedException e) {
+ return false;
+ }
+ return false;
+ }
+
+ /**
+ * Initializes the retry count and retry timeout.
+ * This will determine how many times to retry to acquire lock and the retry timeout.
+ */
+ protected void initRetry() {
+ String retries = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK);
+ try {
+ retryCount = Integer.parseInt(retries);
+ } catch (NumberFormatException e) {
+ retryCount = CarbonCommonConstants.NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK_DEFAULT;
+ }
+
+ String maxTimeout = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK);
+ try {
+ retryTimeout = Integer.parseInt(maxTimeout);
+ } catch (NumberFormatException e) {
+ retryTimeout = CarbonCommonConstants.MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK_DEFAULT;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockFactory.java b/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockFactory.java
new file mode 100644
index 0000000..9890458
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockFactory.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.locks;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+/**
+ * This class is a Lock factory class which is used to provide lock objects.
+ * Using this lock object client can request the lock and unlock.
+ */
+public class CarbonLockFactory {
+
+ /**
+ * lockTypeConfigured to check if zookeeper feature is enabled or not for carbon.
+ */
+ private static String lockTypeConfigured;
+
+ static {
+ CarbonLockFactory.getLockTypeConfigured();
+ }
+
+ /**
+ * This method will determine the lock type.
+ *
+ * @param tableIdentifier
+ * @param lockFile
+ * @return
+ */
+ public static ICarbonLock getCarbonLockObj(CarbonTableIdentifier tableIdentifier,
+ String lockFile) {
+ switch (lockTypeConfigured.toUpperCase()) {
+ case CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL:
+ return new LocalFileLock(tableIdentifier, lockFile);
+
+ case CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER:
+ return new ZooKeeperLocking(tableIdentifier, lockFile);
+
+ case CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS:
+ return new HdfsFileLock(tableIdentifier, lockFile);
+
+ default:
+ throw new UnsupportedOperationException("Not supported the lock type");
+ }
+ }
+
+ /**
+ *
+ * @param locFileLocation
+ * @param lockFile
+ * @return carbon lock
+ */
+ public static ICarbonLock getCarbonLockObj(String locFileLocation, String lockFile) {
+ switch (lockTypeConfigured.toUpperCase()) {
+ case CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL:
+ return new LocalFileLock(locFileLocation, lockFile);
+
+ case CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER:
+ return new ZooKeeperLocking(locFileLocation, lockFile);
+
+ case CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS:
+ return new HdfsFileLock(locFileLocation, lockFile);
+
+ default:
+ throw new UnsupportedOperationException("Not supported the lock type");
+ }
+ }
+
+ /**
+ * This method will set the zookeeper status whether zookeeper to be used for locking or not.
+ */
+ private static void getLockTypeConfigured() {
+ lockTypeConfigured = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.LOCK_TYPE, CarbonCommonConstants.LOCK_TYPE_DEFAULT);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java b/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
new file mode 100644
index 0000000..f8f85e5
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/locks/CarbonLockUtil.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.locks;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+
+/**
+ * This class contains all carbon lock utilities
+ */
+public class CarbonLockUtil {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(CarbonLockUtil.class.getName());
+
+ /**
+ * unlocks given file
+ *
+ * @param carbonLock
+ */
+ public static void fileUnlock(ICarbonLock carbonLock, String locktype) {
+ if (carbonLock.unlock()) {
+ if (locktype.equals(LockUsage.METADATA_LOCK)) {
+ LOGGER.info("Metadata lock has been successfully released");
+ } else if (locktype.equals(LockUsage.TABLE_STATUS_LOCK)) {
+ LOGGER.info("Table status lock has been successfully released");
+ }
+ else if (locktype.equals(LockUsage.CLEAN_FILES_LOCK)) {
+ LOGGER.info("Clean files lock has been successfully released");
+ }
+ else if (locktype.equals(LockUsage.DELETE_SEGMENT_LOCK)) {
+ LOGGER.info("Delete segments lock has been successfully released");
+ }
+ } else {
+ if (locktype.equals(LockUsage.METADATA_LOCK)) {
+ LOGGER.error("Not able to release the metadata lock");
+ } else if (locktype.equals(LockUsage.TABLE_STATUS_LOCK)) {
+ LOGGER.error("Not able to release the table status lock");
+ }
+ else if (locktype.equals(LockUsage.CLEAN_FILES_LOCK)) {
+ LOGGER.info("Not able to release the clean files lock");
+ }
+ else if (locktype.equals(LockUsage.DELETE_SEGMENT_LOCK)) {
+ LOGGER.info("Not able to release the delete segments lock");
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
new file mode 100644
index 0000000..0254dad
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/locks/HdfsFileLock.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.locks;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * This class is used to handle the HDFS File locking.
+ * This is acheived using the concept of acquiring the data out stream using Append option.
+ */
+class HdfsFileLock extends AbstractCarbonLock {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(HdfsFileLock.class.getName());
+ /**
+ * location hdfs file location
+ */
+ private String location;
+
+ private DataOutputStream dataOutputStream;
+
+ public static String tmpPath;
+
+ static {
+ Configuration conf = new Configuration(true);
+ String hdfsPath = conf.get(CarbonCommonConstants.FS_DEFAULT_FS);
+ // By default, we put the hdfs lock meta file for one table inside this table's store folder.
+ // If can not get the STORE_LOCATION, then use hadoop.tmp.dir .
+ tmpPath = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION,
+ System.getProperty(CarbonCommonConstants.HDFS_TEMP_LOCATION));
+ if (!tmpPath.startsWith(CarbonCommonConstants.HDFSURL_PREFIX)) {
+ tmpPath = hdfsPath + tmpPath;
+ }
+ }
+
+ /**
+ * @param lockFileLocation
+ * @param lockFile
+ */
+ public HdfsFileLock(String lockFileLocation, String lockFile) {
+ this.location = tmpPath + CarbonCommonConstants.FILE_SEPARATOR + lockFileLocation
+ + CarbonCommonConstants.FILE_SEPARATOR + lockFile;
+ LOGGER.info("HDFS lock path:"+this.location);
+ initRetry();
+ }
+
+ /**
+ * @param tableIdentifier
+ * @param lockFile
+ */
+ public HdfsFileLock(CarbonTableIdentifier tableIdentifier, String lockFile) {
+ this(tableIdentifier.getDatabaseName() + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier
+ .getTableName(), lockFile);
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.carbondata.core.locks.ICarbonLock#lock()
+ */
+ @Override public boolean lock() {
+ try {
+ if (!FileFactory.isFileExist(location, FileFactory.getFileType(location))) {
+ FileFactory.createNewLockFile(location, FileFactory.getFileType(location));
+ }
+ dataOutputStream =
+ FileFactory.getDataOutputStreamUsingAppend(location, FileFactory.getFileType(location));
+
+ return true;
+
+ } catch (IOException e) {
+ return false;
+ }
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.carbondata.core.locks.ICarbonLock#unlock()
+ */
+ @Override
+ public boolean unlock() {
+ if (null != dataOutputStream) {
+ try {
+ dataOutputStream.close();
+ } catch (IOException e) {
+ return false;
+ } finally {
+ CarbonFile carbonFile =
+ FileFactory.getCarbonFile(location, FileFactory.getFileType(location));
+ if (carbonFile.exists()) {
+ if (carbonFile.delete()) {
+ LOGGER.info("Deleted the lock file " + location);
+ } else {
+ LOGGER.error("Not able to delete the lock file " + location);
+ }
+ } else {
+ LOGGER.error("Not able to delete the lock file because "
+ + "it is not existed in location " + location);
+ }
+ }
+ }
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/core/src/main/java/org/apache/carbondata/core/locks/ICarbonLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/ICarbonLock.java b/core/src/main/java/org/apache/carbondata/core/locks/ICarbonLock.java
new file mode 100644
index 0000000..d984821
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/locks/ICarbonLock.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.locks;
+
+/**
+ * Carbon Lock Interface which handles the locking and unlocking.
+ */
+public interface ICarbonLock {
+
+ /**
+ * Does the unlocking of the acquired lock.
+ *
+ * @return
+ */
+ boolean unlock();
+
+ /**
+ * This will acquire the lock and if it doesnt get then it will retry after the confiured time.
+ *
+ * @return
+ */
+ boolean lockWithRetries();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
new file mode 100644
index 0000000..fccb080
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/locks/LocalFileLock.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.locks;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.channels.OverlappingFileLockException;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+
+/**
+ * This class handles the file locking in the local file system.
+ * This will be handled using the file channel lock API.
+ */
+class LocalFileLock extends AbstractCarbonLock {
+ /**
+ * location is the location of the lock file.
+ */
+ private String location;
+
+ /**
+ * fileOutputStream of the local lock file
+ */
+ private FileOutputStream fileOutputStream;
+
+ /**
+ * channel is the FileChannel of the lock file.
+ */
+ private FileChannel channel;
+
+ /**
+ * fileLock NIO FileLock Object
+ */
+ private FileLock fileLock;
+
+ /**
+ * lock file
+ */
+ private String lockFile;
+
+ public static final String tmpPath;
+
+ private String lockFilePath;
+
+ /**
+ * LOGGER for logging the messages.
+ */
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(LocalFileLock.class.getName());
+
+ static {
+ tmpPath = System.getProperty("java.io.tmpdir");
+ }
+
+ /**
+ * @param lockFileLocation
+ * @param lockFile
+ */
+ public LocalFileLock(String lockFileLocation, String lockFile) {
+ this.location = tmpPath + CarbonCommonConstants.FILE_SEPARATOR + lockFileLocation;
+ this.lockFile = lockFile;
+ initRetry();
+ }
+
+ /**
+ * @param tableIdentifier
+ * @param lockFile
+ */
+ public LocalFileLock(CarbonTableIdentifier tableIdentifier, String lockFile) {
+ this(tableIdentifier.getDatabaseName() + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier
+ .getTableName(), lockFile);
+ initRetry();
+ }
+
+ /**
+ * Lock API for locking of the file channel of the lock file.
+ *
+ * @return
+ */
+ @Override public boolean lock() {
+ try {
+ if (!FileFactory.isFileExist(location, FileFactory.getFileType(tmpPath))) {
+ FileFactory.mkdirs(location, FileFactory.getFileType(tmpPath));
+ }
+ lockFilePath = location + CarbonCommonConstants.FILE_SEPARATOR +
+ lockFile;
+ if (!FileFactory.isFileExist(lockFilePath, FileFactory.getFileType(location))) {
+ FileFactory.createNewLockFile(lockFilePath, FileFactory.getFileType(location));
+ }
+
+ fileOutputStream = new FileOutputStream(lockFilePath);
+ channel = fileOutputStream.getChannel();
+ try {
+ fileLock = channel.tryLock();
+ } catch (OverlappingFileLockException e) {
+ return false;
+ }
+ if (null != fileLock) {
+ return true;
+ } else {
+ return false;
+ }
+ } catch (IOException e) {
+ return false;
+ }
+
+ }
+
+ /**
+ * Unlock API for unlocking of the acquired lock.
+ *
+ * @return
+ */
+ @Override public boolean unlock() {
+ boolean status;
+ try {
+ if (null != fileLock) {
+ fileLock.release();
+ }
+ status = true;
+ } catch (IOException e) {
+ status = false;
+ } finally {
+ if (null != fileOutputStream) {
+ try {
+ fileOutputStream.close();
+ // deleting the lock file after releasing the lock.
+ if (FileFactory.getCarbonFile(lockFilePath, FileFactory.getFileType(lockFilePath))
+ .delete()) {
+ LOGGER.info("Successfully deleted the lock file " + lockFilePath);
+ } else {
+ LOGGER.error("Not able to delete the lock file " + lockFilePath);
+ }
+ } catch (IOException e) {
+ LOGGER.error(e.getMessage());
+ }
+ }
+ }
+ return status;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/core/src/main/java/org/apache/carbondata/core/locks/LockUsage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/LockUsage.java b/core/src/main/java/org/apache/carbondata/core/locks/LockUsage.java
new file mode 100644
index 0000000..1c6ef8e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/locks/LockUsage.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.locks;
+
+/**
+ * This enum is used to define the usecase of the lock.
+ * Each enum value is one specific lock case.
+ */
+public class LockUsage {
+ public static final String LOCK = ".lock";
+ public static final String METADATA_LOCK = "meta.lock";
+ public static final String COMPACTION_LOCK = "compaction.lock";
+ public static final String SYSTEMLEVEL_COMPACTION_LOCK = "system_level_compaction.lock";
+ public static final String TABLE_STATUS_LOCK = "tablestatus.lock";
+ public static final String TABLE_UPDATE_STATUS_LOCK = "tableupdatestatus.lock";
+ public static final String DELETE_SEGMENT_LOCK = "delete_segment.lock";
+ public static final String CLEAN_FILES_LOCK = "clean_files.lock";
+ public static final String DROP_TABLE_LOCK = "droptable.lock";
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/core/src/main/java/org/apache/carbondata/core/locks/ZooKeeperLocking.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/ZooKeeperLocking.java b/core/src/main/java/org/apache/carbondata/core/locks/ZooKeeperLocking.java
new file mode 100644
index 0000000..ff7b67d
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/locks/ZooKeeperLocking.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.locks;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.ZooKeeper;
+
+/**
+ * For Handling the zookeeper locking implementation
+ */
+class ZooKeeperLocking extends AbstractCarbonLock {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(ZooKeeperLocking.class.getName());
+
+ /**
+ * zk is the zookeeper client instance
+ */
+ private static ZooKeeper zk;
+
+ /**
+ * zooKeeperLocation is the location in the zoo keeper file system where the locks will be
+ * maintained.
+ */
+ private static final String zooKeeperLocation = CarbonCommonConstants.ZOOKEEPER_LOCATION;
+
+ /**
+ * Unique folder for each table with DatabaseName_TableName
+ */
+ private final String tableIdFolder;
+
+ /**
+ * lockName is the name of the lock to use. This name should be same for every process that want
+ * to share the same lock
+ */
+ private String lockName;
+
+ /**
+ * lockPath is the unique path created for the each instance of the carbon lock.
+ */
+ private String lockPath;
+
+ private String lockTypeFolder;
+
+ public ZooKeeperLocking(CarbonTableIdentifier tableIdentifier, String lockFile) {
+ this(tableIdentifier.getDatabaseName() + File.separator + tableIdentifier.getTableName(),
+ lockFile);
+ }
+
+ /**
+ * @param lockLocation
+ * @param lockFile
+ */
+ public ZooKeeperLocking(String lockLocation, String lockFile) {
+ this.lockName = lockFile;
+ this.tableIdFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + lockLocation;
+
+ String zooKeeperUrl =
+ CarbonProperties.getInstance().getProperty(CarbonCommonConstants.ZOOKEEPER_URL);
+ zk = ZookeeperInit.getInstance(zooKeeperUrl).getZookeeper();
+
+ this.lockTypeFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + lockLocation
+ + CarbonCommonConstants.FILE_SEPARATOR + lockFile;
+ try {
+ createBaseNode();
+ // if exists returns null then path doesnt exist. so creating.
+ if (null == zk.exists(this.tableIdFolder, true)) {
+ createRecursivly(this.tableIdFolder);
+ }
+ // if exists returns null then path doesnt exist. so creating.
+ if (null == zk.exists(this.lockTypeFolder, true)) {
+ zk.create(this.lockTypeFolder, new byte[1], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+ }
+ } catch (KeeperException | InterruptedException e) {
+ LOGGER.error(e, e.getMessage());
+ }
+ initRetry();
+ }
+
+ /**
+ * Creating a znode in which all the znodes (lock files )are maintained.
+ */
+ private void createBaseNode() throws KeeperException, InterruptedException {
+ if (null == zk.exists(zooKeeperLocation, true)) {
+ // creating a znode in which all the znodes (lock files )are maintained.
+ zk.create(zooKeeperLocation, new byte[1], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+ }
+ }
+
+ /**
+ * Create zookeepr node if not exist
+ * @param path
+ * @throws KeeperException
+ * @throws InterruptedException
+ */
+ private void createRecursivly(String path) throws KeeperException, InterruptedException {
+ try {
+ if (zk.exists(path, true) == null && path.length() > 0) {
+ String temp = path.substring(0, path.lastIndexOf(File.separator));
+ createRecursivly(temp);
+ zk.create(path, null, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+ } else {
+ return;
+ }
+ } catch (KeeperException e) {
+ throw e;
+ } catch (InterruptedException e) {
+ throw e;
+ }
+
+ }
+ /**
+ * Handling of the locking mechanism using zoo keeper.
+ */
+ @Override public boolean lock() {
+ try {
+ // create the lock file with lockName.
+ lockPath =
+ zk.create(this.lockTypeFolder + CarbonCommonConstants.FILE_SEPARATOR + lockName, null,
+ Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL);
+
+ // get the children present in zooKeeperLocation.
+ List<String> nodes = zk.getChildren(this.lockTypeFolder, null);
+
+ // sort the childrens
+ Collections.sort(nodes);
+
+ // here the logic is , for each lock request zookeeper will create a file ending with
+ // incremental digits.
+ // so first request will be 00001 next is 00002 and so on.
+ // if the current request is 00002 and already one previous request(00001) is present then get
+ // children will give both nodes.
+ // after the sort we are checking if the lock path is first or not .if it is first then lock
+ // has been acquired.
+
+ if (lockPath.endsWith(nodes.get(0))) {
+ return true;
+ } else {
+ // if locking failed then deleting the created lock as next time again new lock file will be
+ // created.
+ zk.delete(lockPath, -1);
+ return false;
+ }
+ } catch (KeeperException | InterruptedException e) {
+ LOGGER.error(e, e.getMessage());
+ return false;
+ }
+ }
+
+ /**
+ * @return status where lock file is unlocked or not.
+ */
+ @Override public boolean unlock() {
+ try {
+ // exists will return null if the path doesn't exists.
+ if (null != zk.exists(lockPath, true)) {
+ zk.delete(lockPath, -1);
+ lockPath = null;
+ }
+ } catch (KeeperException | InterruptedException e) {
+ LOGGER.error(e, e.getMessage());
+ return false;
+ }
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/core/src/main/java/org/apache/carbondata/core/locks/ZookeeperInit.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/locks/ZookeeperInit.java b/core/src/main/java/org/apache/carbondata/core/locks/ZookeeperInit.java
new file mode 100644
index 0000000..3b6e0b8
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/locks/ZookeeperInit.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.locks;
+
+import java.io.IOException;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+
+/**
+ * This is a singleton class for initialization of zookeeper client.
+ */
+public class ZookeeperInit {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(ZookeeperInit.class.getName());
+
+ private static ZookeeperInit zooKeeperInit;
+ /**
+ * zk is the zookeeper client instance
+ */
+ private ZooKeeper zk;
+
+ private ZookeeperInit(String zooKeeperUrl) {
+
+ int sessionTimeOut = 100000;
+ try {
+ zk = new ZooKeeper(zooKeeperUrl, sessionTimeOut, new DummyWatcher());
+
+ } catch (IOException e) {
+ LOGGER.error(e.getMessage());
+ }
+
+ }
+
+ public static ZookeeperInit getInstance(String zooKeeperUrl) {
+
+ if (null == zooKeeperInit) {
+ synchronized (ZookeeperInit.class) {
+ if (null == zooKeeperInit) {
+ LOGGER.info("Initiating Zookeeper client.");
+ zooKeeperInit = new ZookeeperInit(zooKeeperUrl);
+ }
+ }
+ }
+ return zooKeeperInit;
+
+ }
+
+ public static ZookeeperInit getInstance() {
+ return zooKeeperInit;
+ }
+
+ public ZooKeeper getZookeeper() {
+ return zk;
+ }
+
+ private static class DummyWatcher implements Watcher {
+ public void process(WatchedEvent event) {
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java
index d9d5651..37869da 100644
--- a/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java
@@ -31,6 +31,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.locks.ICarbonLock;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.path.CarbonStorePath;
@@ -40,7 +41,6 @@ import org.apache.carbondata.core.update.data.RowCountDetailsVO;
import org.apache.carbondata.core.updatestatus.LoadMetadataDetails;
import org.apache.carbondata.core.updatestatus.SegmentStatusManager;
import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager;
-import org.apache.carbondata.core.updatestatus.locks.ICarbonLock;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java
index c43369e..7913c17 100644
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java
@@ -38,15 +38,15 @@ import org.apache.carbondata.core.atomic.AtomicFileOperationsImpl;
import org.apache.carbondata.core.atomic.FileWriteOperation;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.locks.CarbonLockFactory;
+import org.apache.carbondata.core.locks.CarbonLockUtil;
+import org.apache.carbondata.core.locks.ICarbonLock;
+import org.apache.carbondata.core.locks.LockUsage;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.update.CarbonUpdateUtil;
-import org.apache.carbondata.core.updatestatus.locks.CarbonLockFactory;
-import org.apache.carbondata.core.updatestatus.locks.CarbonLockUtil;
-import org.apache.carbondata.core.updatestatus.locks.ICarbonLock;
-import org.apache.carbondata.core.updatestatus.locks.LockUsage;
import org.apache.carbondata.core.util.CarbonUtil;
import com.google.gson.Gson;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java
index 371e6f9..990661d 100644
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java
@@ -39,6 +39,9 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.locks.CarbonLockFactory;
+import org.apache.carbondata.core.locks.ICarbonLock;
+import org.apache.carbondata.core.locks.LockUsage;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
@@ -47,9 +50,6 @@ import org.apache.carbondata.core.update.CarbonUpdateUtil;
import org.apache.carbondata.core.update.SegmentUpdateDetails;
import org.apache.carbondata.core.update.TupleIdEnum;
import org.apache.carbondata.core.update.data.UpdateVO;
-import org.apache.carbondata.core.updatestatus.locks.CarbonLockFactory;
-import org.apache.carbondata.core.updatestatus.locks.ICarbonLock;
-import org.apache.carbondata.core.updatestatus.locks.LockUsage;
import org.apache.carbondata.core.util.CarbonUtil;
import com.google.gson.Gson;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/AbstractCarbonLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/AbstractCarbonLock.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/AbstractCarbonLock.java
deleted file mode 100644
index 35f5aee..0000000
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/AbstractCarbonLock.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.updatestatus.locks;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.util.CarbonProperties;
-
-/**
- * This is the abstract class of the lock implementations.This handles the
- * retrying part of the locking.
- */
-abstract class AbstractCarbonLock implements ICarbonLock {
- private int retryCount;
-
- private int retryTimeout;
-
- public abstract boolean lock();
-
- /**
- * API for enabling the locking of file with retries.
- */
- public boolean lockWithRetries() {
- try {
- for (int i = 0; i < retryCount; i++) {
- if (lock()) {
- return true;
- } else {
- Thread.sleep(retryTimeout * 1000L);
- }
- }
- } catch (InterruptedException e) {
- return false;
- }
- return false;
- }
-
- /**
- * Initializes the retry count and retry timeout.
- * This will determine how many times to retry to acquire lock and the retry timeout.
- */
- protected void initRetry() {
- String retries = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK);
- try {
- retryCount = Integer.parseInt(retries);
- } catch (NumberFormatException e) {
- retryCount = CarbonCommonConstants.NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK_DEFAULT;
- }
-
- String maxTimeout = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK);
- try {
- retryTimeout = Integer.parseInt(maxTimeout);
- } catch (NumberFormatException e) {
- retryTimeout = CarbonCommonConstants.MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK_DEFAULT;
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockFactory.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockFactory.java
deleted file mode 100644
index 228d59c..0000000
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockFactory.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.updatestatus.locks;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
-import org.apache.carbondata.core.util.CarbonProperties;
-
-/**
- * This class is a Lock factory class which is used to provide lock objects.
- * Using this lock object client can request the lock and unlock.
- */
-public class CarbonLockFactory {
-
- /**
- * lockTypeConfigured to check if zookeeper feature is enabled or not for carbon.
- */
- private static String lockTypeConfigured;
-
- static {
- CarbonLockFactory.getLockTypeConfigured();
- }
-
- /**
- * This method will determine the lock type.
- *
- * @param tableIdentifier
- * @param lockFile
- * @return
- */
- public static ICarbonLock getCarbonLockObj(CarbonTableIdentifier tableIdentifier,
- String lockFile) {
- switch (lockTypeConfigured.toUpperCase()) {
- case CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL:
- return new LocalFileLock(tableIdentifier, lockFile);
-
- case CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER:
- return new ZooKeeperLocking(tableIdentifier, lockFile);
-
- case CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS:
- return new HdfsFileLock(tableIdentifier, lockFile);
-
- default:
- throw new UnsupportedOperationException("Not supported the lock type");
- }
- }
-
- /**
- *
- * @param locFileLocation
- * @param lockFile
- * @return carbon lock
- */
- public static ICarbonLock getCarbonLockObj(String locFileLocation, String lockFile) {
- switch (lockTypeConfigured.toUpperCase()) {
- case CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL:
- return new LocalFileLock(locFileLocation, lockFile);
-
- case CarbonCommonConstants.CARBON_LOCK_TYPE_ZOOKEEPER:
- return new ZooKeeperLocking(locFileLocation, lockFile);
-
- case CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS:
- return new HdfsFileLock(locFileLocation, lockFile);
-
- default:
- throw new UnsupportedOperationException("Not supported the lock type");
- }
- }
-
- /**
- * This method will set the zookeeper status whether zookeeper to be used for locking or not.
- */
- private static void getLockTypeConfigured() {
- lockTypeConfigured = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.LOCK_TYPE, CarbonCommonConstants.LOCK_TYPE_DEFAULT);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockUtil.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockUtil.java
deleted file mode 100644
index b73bedd..0000000
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/CarbonLockUtil.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.updatestatus.locks;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-
-/**
- * This class contains all carbon lock utilities
- */
-public class CarbonLockUtil {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CarbonLockUtil.class.getName());
-
- /**
- * unlocks given file
- *
- * @param carbonLock
- */
- public static void fileUnlock(ICarbonLock carbonLock, String locktype) {
- if (carbonLock.unlock()) {
- if (locktype.equals(LockUsage.METADATA_LOCK)) {
- LOGGER.info("Metadata lock has been successfully released");
- } else if (locktype.equals(LockUsage.TABLE_STATUS_LOCK)) {
- LOGGER.info("Table status lock has been successfully released");
- }
- else if (locktype.equals(LockUsage.CLEAN_FILES_LOCK)) {
- LOGGER.info("Clean files lock has been successfully released");
- }
- else if (locktype.equals(LockUsage.DELETE_SEGMENT_LOCK)) {
- LOGGER.info("Delete segments lock has been successfully released");
- }
- } else {
- if (locktype.equals(LockUsage.METADATA_LOCK)) {
- LOGGER.error("Not able to release the metadata lock");
- } else if (locktype.equals(LockUsage.TABLE_STATUS_LOCK)) {
- LOGGER.error("Not able to release the table status lock");
- }
- else if (locktype.equals(LockUsage.CLEAN_FILES_LOCK)) {
- LOGGER.info("Not able to release the clean files lock");
- }
- else if (locktype.equals(LockUsage.DELETE_SEGMENT_LOCK)) {
- LOGGER.info("Not able to release the delete segments lock");
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/HdfsFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/HdfsFileLock.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/HdfsFileLock.java
deleted file mode 100644
index ede5883..0000000
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/HdfsFileLock.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.updatestatus.locks;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
-import org.apache.carbondata.core.util.CarbonProperties;
-
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * This class is used to handle the HDFS File locking.
- * This is acheived using the concept of acquiring the data out stream using Append option.
- */
-class HdfsFileLock extends AbstractCarbonLock {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(HdfsFileLock.class.getName());
- /**
- * location hdfs file location
- */
- private String location;
-
- private DataOutputStream dataOutputStream;
-
- public static String tmpPath;
-
- static {
- Configuration conf = new Configuration(true);
- String hdfsPath = conf.get(CarbonCommonConstants.FS_DEFAULT_FS);
- // By default, we put the hdfs lock meta file for one table inside this table's store folder.
- // If can not get the STORE_LOCATION, then use hadoop.tmp.dir .
- tmpPath = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION,
- System.getProperty(CarbonCommonConstants.HDFS_TEMP_LOCATION));
- if (!tmpPath.startsWith(CarbonCommonConstants.HDFSURL_PREFIX)) {
- tmpPath = hdfsPath + tmpPath;
- }
- }
-
- /**
- * @param lockFileLocation
- * @param lockFile
- */
- public HdfsFileLock(String lockFileLocation, String lockFile) {
- this.location = tmpPath + CarbonCommonConstants.FILE_SEPARATOR + lockFileLocation
- + CarbonCommonConstants.FILE_SEPARATOR + lockFile;
- LOGGER.info("HDFS lock path:"+this.location);
- initRetry();
- }
-
- /**
- * @param tableIdentifier
- * @param lockFile
- */
- public HdfsFileLock(CarbonTableIdentifier tableIdentifier, String lockFile) {
- this(tableIdentifier.getDatabaseName() + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier
- .getTableName(), lockFile);
- }
-
- /* (non-Javadoc)
- * @see org.apache.carbondata.core.locks.ICarbonLock#lock()
- */
- @Override public boolean lock() {
- try {
- if (!FileFactory.isFileExist(location, FileFactory.getFileType(location))) {
- FileFactory.createNewLockFile(location, FileFactory.getFileType(location));
- }
- dataOutputStream =
- FileFactory.getDataOutputStreamUsingAppend(location, FileFactory.getFileType(location));
-
- return true;
-
- } catch (IOException e) {
- return false;
- }
- }
-
- /* (non-Javadoc)
- * @see org.apache.carbondata.core.locks.ICarbonLock#unlock()
- */
- @Override
- public boolean unlock() {
- if (null != dataOutputStream) {
- try {
- dataOutputStream.close();
- } catch (IOException e) {
- return false;
- } finally {
- CarbonFile carbonFile =
- FileFactory.getCarbonFile(location, FileFactory.getFileType(location));
- if (carbonFile.exists()) {
- if (carbonFile.delete()) {
- LOGGER.info("Deleted the lock file " + location);
- } else {
- LOGGER.error("Not able to delete the lock file " + location);
- }
- } else {
- LOGGER.error("Not able to delete the lock file because "
- + "it is not existed in location " + location);
- }
- }
- }
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ICarbonLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ICarbonLock.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ICarbonLock.java
deleted file mode 100644
index e269c86..0000000
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ICarbonLock.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.updatestatus.locks;
-
-/**
- * Carbon Lock Interface which handles the locking and unlocking.
- */
-public interface ICarbonLock {
-
- /**
- * Does the unlocking of the acquired lock.
- *
- * @return
- */
- boolean unlock();
-
- /**
- * This will acquire the lock and if it doesnt get then it will retry after the confiured time.
- *
- * @return
- */
- boolean lockWithRetries();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLock.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLock.java
deleted file mode 100644
index 1691b37..0000000
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLock.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.updatestatus.locks;
-
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.channels.FileChannel;
-import java.nio.channels.FileLock;
-import java.nio.channels.OverlappingFileLockException;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastore.impl.FileFactory;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
-
-/**
- * This class handles the file locking in the local file system.
- * This will be handled using the file channel lock API.
- */
-class LocalFileLock extends AbstractCarbonLock {
- /**
- * location is the location of the lock file.
- */
- private String location;
-
- /**
- * fileOutputStream of the local lock file
- */
- private FileOutputStream fileOutputStream;
-
- /**
- * channel is the FileChannel of the lock file.
- */
- private FileChannel channel;
-
- /**
- * fileLock NIO FileLock Object
- */
- private FileLock fileLock;
-
- /**
- * lock file
- */
- private String lockFile;
-
- public static final String tmpPath;
-
- private String lockFilePath;
-
- /**
- * LOGGER for logging the messages.
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(LocalFileLock.class.getName());
-
- static {
- tmpPath = System.getProperty("java.io.tmpdir");
- }
-
- /**
- * @param lockFileLocation
- * @param lockFile
- */
- public LocalFileLock(String lockFileLocation, String lockFile) {
- this.location = tmpPath + CarbonCommonConstants.FILE_SEPARATOR + lockFileLocation;
- this.lockFile = lockFile;
- initRetry();
- }
-
- /**
- * @param tableIdentifier
- * @param lockFile
- */
- public LocalFileLock(CarbonTableIdentifier tableIdentifier, String lockFile) {
- this(tableIdentifier.getDatabaseName() + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier
- .getTableName(), lockFile);
- initRetry();
- }
-
- /**
- * Lock API for locking of the file channel of the lock file.
- *
- * @return
- */
- @Override public boolean lock() {
- try {
- if (!FileFactory.isFileExist(location, FileFactory.getFileType(tmpPath))) {
- FileFactory.mkdirs(location, FileFactory.getFileType(tmpPath));
- }
- lockFilePath = location + CarbonCommonConstants.FILE_SEPARATOR +
- lockFile;
- if (!FileFactory.isFileExist(lockFilePath, FileFactory.getFileType(location))) {
- FileFactory.createNewLockFile(lockFilePath, FileFactory.getFileType(location));
- }
-
- fileOutputStream = new FileOutputStream(lockFilePath);
- channel = fileOutputStream.getChannel();
- try {
- fileLock = channel.tryLock();
- } catch (OverlappingFileLockException e) {
- return false;
- }
- if (null != fileLock) {
- return true;
- } else {
- return false;
- }
- } catch (IOException e) {
- return false;
- }
-
- }
-
- /**
- * Unlock API for unlocking of the acquired lock.
- *
- * @return
- */
- @Override public boolean unlock() {
- boolean status;
- try {
- if (null != fileLock) {
- fileLock.release();
- }
- status = true;
- } catch (IOException e) {
- status = false;
- } finally {
- if (null != fileOutputStream) {
- try {
- fileOutputStream.close();
- // deleting the lock file after releasing the lock.
- if (FileFactory.getCarbonFile(lockFilePath, FileFactory.getFileType(lockFilePath))
- .delete()) {
- LOGGER.info("Successfully deleted the lock file " + lockFilePath);
- } else {
- LOGGER.error("Not able to delete the lock file " + lockFilePath);
- }
- } catch (IOException e) {
- LOGGER.error(e.getMessage());
- }
- }
- }
- return status;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LockUsage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LockUsage.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LockUsage.java
deleted file mode 100644
index 5a2ffe7..0000000
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LockUsage.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.updatestatus.locks;
-
-/**
- * This enum is used to define the usecase of the lock.
- * Each enum value is one specific lock case.
- */
-public class LockUsage {
- public static final String LOCK = ".lock";
- public static final String METADATA_LOCK = "meta.lock";
- public static final String COMPACTION_LOCK = "compaction.lock";
- public static final String SYSTEMLEVEL_COMPACTION_LOCK = "system_level_compaction.lock";
- public static final String TABLE_STATUS_LOCK = "tablestatus.lock";
- public static final String TABLE_UPDATE_STATUS_LOCK = "tableupdatestatus.lock";
- public static final String DELETE_SEGMENT_LOCK = "delete_segment.lock";
- public static final String CLEAN_FILES_LOCK = "clean_files.lock";
- public static final String DROP_TABLE_LOCK = "droptable.lock";
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZooKeeperLocking.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZooKeeperLocking.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZooKeeperLocking.java
deleted file mode 100644
index 5e750b5..0000000
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZooKeeperLocking.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.updatestatus.locks;
-
-import java.io.File;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
-import org.apache.carbondata.core.util.CarbonProperties;
-
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.ZooDefs.Ids;
-import org.apache.zookeeper.ZooKeeper;
-
-/**
- * For Handling the zookeeper locking implementation
- */
-class ZooKeeperLocking extends AbstractCarbonLock {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(ZooKeeperLocking.class.getName());
-
- /**
- * zk is the zookeeper client instance
- */
- private static ZooKeeper zk;
-
- /**
- * zooKeeperLocation is the location in the zoo keeper file system where the locks will be
- * maintained.
- */
- private static final String zooKeeperLocation = CarbonCommonConstants.ZOOKEEPER_LOCATION;
-
- /**
- * Unique folder for each table with DatabaseName_TableName
- */
- private final String tableIdFolder;
-
- /**
- * lockName is the name of the lock to use. This name should be same for every process that want
- * to share the same lock
- */
- private String lockName;
-
- /**
- * lockPath is the unique path created for the each instance of the carbon lock.
- */
- private String lockPath;
-
- private String lockTypeFolder;
-
- public ZooKeeperLocking(CarbonTableIdentifier tableIdentifier, String lockFile) {
- this(tableIdentifier.getDatabaseName() + File.separator + tableIdentifier.getTableName(),
- lockFile);
- }
-
- /**
- * @param lockLocation
- * @param lockFile
- */
- public ZooKeeperLocking(String lockLocation, String lockFile) {
- this.lockName = lockFile;
- this.tableIdFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + lockLocation;
-
- String zooKeeperUrl =
- CarbonProperties.getInstance().getProperty(CarbonCommonConstants.ZOOKEEPER_URL);
- zk = ZookeeperInit.getInstance(zooKeeperUrl).getZookeeper();
-
- this.lockTypeFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + lockLocation
- + CarbonCommonConstants.FILE_SEPARATOR + lockFile;
- try {
- createBaseNode();
- // if exists returns null then path doesnt exist. so creating.
- if (null == zk.exists(this.tableIdFolder, true)) {
- createRecursivly(this.tableIdFolder);
- }
- // if exists returns null then path doesnt exist. so creating.
- if (null == zk.exists(this.lockTypeFolder, true)) {
- zk.create(this.lockTypeFolder, new byte[1], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- }
- } catch (KeeperException | InterruptedException e) {
- LOGGER.error(e, e.getMessage());
- }
- initRetry();
- }
-
- /**
- * Creating a znode in which all the znodes (lock files )are maintained.
- */
- private void createBaseNode() throws KeeperException, InterruptedException {
- if (null == zk.exists(zooKeeperLocation, true)) {
- // creating a znode in which all the znodes (lock files )are maintained.
- zk.create(zooKeeperLocation, new byte[1], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- }
- }
-
- /**
- * Create zookeepr node if not exist
- * @param path
- * @throws KeeperException
- * @throws InterruptedException
- */
- private void createRecursivly(String path) throws KeeperException, InterruptedException {
- try {
- if (zk.exists(path, true) == null && path.length() > 0) {
- String temp = path.substring(0, path.lastIndexOf(File.separator));
- createRecursivly(temp);
- zk.create(path, null, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- } else {
- return;
- }
- } catch (KeeperException e) {
- throw e;
- } catch (InterruptedException e) {
- throw e;
- }
-
- }
- /**
- * Handling of the locking mechanism using zoo keeper.
- */
- @Override public boolean lock() {
- try {
- // create the lock file with lockName.
- lockPath =
- zk.create(this.lockTypeFolder + CarbonCommonConstants.FILE_SEPARATOR + lockName, null,
- Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL);
-
- // get the children present in zooKeeperLocation.
- List<String> nodes = zk.getChildren(this.lockTypeFolder, null);
-
- // sort the childrens
- Collections.sort(nodes);
-
- // here the logic is , for each lock request zookeeper will create a file ending with
- // incremental digits.
- // so first request will be 00001 next is 00002 and so on.
- // if the current request is 00002 and already one previous request(00001) is present then get
- // children will give both nodes.
- // after the sort we are checking if the lock path is first or not .if it is first then lock
- // has been acquired.
-
- if (lockPath.endsWith(nodes.get(0))) {
- return true;
- } else {
- // if locking failed then deleting the created lock as next time again new lock file will be
- // created.
- zk.delete(lockPath, -1);
- return false;
- }
- } catch (KeeperException | InterruptedException e) {
- LOGGER.error(e, e.getMessage());
- return false;
- }
- }
-
- /**
- * @return status where lock file is unlocked or not.
- */
- @Override public boolean unlock() {
- try {
- // exists will return null if the path doesn't exists.
- if (null != zk.exists(lockPath, true)) {
- zk.delete(lockPath, -1);
- lockPath = null;
- }
- } catch (KeeperException | InterruptedException e) {
- LOGGER.error(e, e.getMessage());
- return false;
- }
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZookeeperInit.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZookeeperInit.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZookeeperInit.java
deleted file mode 100644
index 391e174..0000000
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/ZookeeperInit.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.updatestatus.locks;
-
-import java.io.IOException;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.ZooKeeper;
-
-/**
- * This is a singleton class for initialization of zookeeper client.
- */
-public class ZookeeperInit {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(ZookeeperInit.class.getName());
-
- private static ZookeeperInit zooKeeperInit;
- /**
- * zk is the zookeeper client instance
- */
- private ZooKeeper zk;
-
- private ZookeeperInit(String zooKeeperUrl) {
-
- int sessionTimeOut = 100000;
- try {
- zk = new ZooKeeper(zooKeeperUrl, sessionTimeOut, new DummyWatcher());
-
- } catch (IOException e) {
- LOGGER.error(e.getMessage());
- }
-
- }
-
- public static ZookeeperInit getInstance(String zooKeeperUrl) {
-
- if (null == zooKeeperInit) {
- synchronized (ZookeeperInit.class) {
- if (null == zooKeeperInit) {
- LOGGER.info("Initiating Zookeeper client.");
- zooKeeperInit = new ZookeeperInit(zooKeeperUrl);
- }
- }
- }
- return zooKeeperInit;
-
- }
-
- public static ZookeeperInit getInstance() {
- return zooKeeperInit;
- }
-
- public ZooKeeper getZookeeper() {
- return zk;
- }
-
- private static class DummyWatcher implements Watcher {
- public void process(WatchedEvent event) {
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/core/src/test/java/org/apache/carbondata/core/locks/LocalFileLockTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/locks/LocalFileLockTest.java b/core/src/test/java/org/apache/carbondata/core/locks/LocalFileLockTest.java
new file mode 100644
index 0000000..1d1a0ae
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/locks/LocalFileLockTest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.locks;
+
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.pentaho.di.core.util.Assert;
+
+/**
+ * Test class to test the functionality of the local file locking.
+ *
+ * @author Administrator
+ */
+public class LocalFileLockTest {
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @Before public void setUp() throws Exception {
+ }
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @After public void tearDown() throws Exception {
+ }
+
+ @Test public void testingLocalFileLockingByAcquiring2Locks() {
+
+ CarbonTableIdentifier carbonTableIdentifier = new CarbonTableIdentifier("databaseName", "tableName", "tableId");
+ LocalFileLock localLock1 =
+ new LocalFileLock(carbonTableIdentifier,
+ LockUsage.METADATA_LOCK);
+ Assert.assertTrue(localLock1.lock());
+ LocalFileLock localLock2 =
+ new LocalFileLock(carbonTableIdentifier,
+ LockUsage.METADATA_LOCK);
+ Assert.assertTrue(!localLock2.lock());
+
+ Assert.assertTrue(localLock1.unlock());
+ Assert.assertTrue(localLock2.lock());
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/core/src/test/java/org/apache/carbondata/core/locks/ZooKeeperLockingTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/locks/ZooKeeperLockingTest.java b/core/src/test/java/org/apache/carbondata/core/locks/ZooKeeperLockingTest.java
new file mode 100644
index 0000000..4ea7227
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/locks/ZooKeeperLockingTest.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.locks;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.util.Properties;
+
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+import mockit.NonStrictExpectations;
+import org.apache.zookeeper.server.ServerConfig;
+import org.apache.zookeeper.server.ZooKeeperServerMain;
+import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * @author Administrator
+ */
+public class ZooKeeperLockingTest {
+
+ int freePort;
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @Before public void setUp() throws Exception {
+ Properties startupProperties = new Properties();
+ startupProperties.setProperty("dataDir", (new File("./target").getAbsolutePath()));
+ startupProperties.setProperty("dataLogDir", (new File("./target").getAbsolutePath()));
+ freePort = findFreePort();
+ startupProperties.setProperty("clientPort", "" + freePort);
+ QuorumPeerConfig quorumConfiguration = new QuorumPeerConfig();
+ try {
+ quorumConfiguration.parseProperties(startupProperties);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ final ZooKeeperServerMain zooKeeperServer = new ZooKeeperServerMain();
+ final ServerConfig configuration = new ServerConfig();
+ configuration.readFrom(quorumConfiguration);
+ new Thread() {
+ public void run() {
+ try {
+ zooKeeperServer.runFromConfig(configuration);
+ } catch (IOException e) {
+ System.out.println("ZooKeeper failure");
+ }
+ }
+ }.start();
+ }
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @After public void tearDown() throws Exception {
+ }
+
+ @Test public void testZooKeeperLockingByTryingToAcquire2Locks()
+ throws IllegalArgumentException, IllegalAccessException, NoSuchFieldException,
+ SecurityException {
+
+ final CarbonProperties cp = CarbonProperties.getInstance();
+ new NonStrictExpectations(cp) {
+ {
+ cp.getProperty("/CarbonLocks");
+ result = "/carbontests";
+ cp.getProperty("spark.deploy.zookeeper.url");
+ result = "127.0.0.1:" + freePort;
+ }
+ };
+
+ ZookeeperInit zki = ZookeeperInit.getInstance("127.0.0.1:" + freePort);
+
+ CarbonTableIdentifier tableIdentifier = new CarbonTableIdentifier("dbName", "tableName", "tableId");
+ ZooKeeperLocking zkl =
+ new ZooKeeperLocking(tableIdentifier,
+ LockUsage.METADATA_LOCK);
+ Assert.assertTrue(zkl.lock());
+
+ ZooKeeperLocking zk2 = new ZooKeeperLocking(
+ tableIdentifier, LockUsage.METADATA_LOCK);
+ Assert.assertTrue(!zk2.lock());
+
+ Assert.assertTrue(zkl.unlock());
+ Assert.assertTrue(zk2.lock());
+ Assert.assertTrue(zk2.unlock());
+ }
+
+ /**
+ * For finding the free port available.
+ *
+ * @return
+ */
+ private static int findFreePort() {
+ ServerSocket socket = null;
+ try {
+ socket = new ServerSocket(0);
+ socket.setReuseAddress(true);
+ int port = socket.getLocalPort();
+ try {
+ socket.close();
+ } catch (IOException e) {
+ // Ignore IOException on close()
+ }
+ return port;
+ } catch (Exception e) {
+ // Ignore
+ } finally {
+ if (socket != null) {
+ try {
+ socket.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ return 2181;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/afa88ee6/core/src/test/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLockTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLockTest.java b/core/src/test/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLockTest.java
deleted file mode 100644
index d69b6be..0000000
--- a/core/src/test/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLockTest.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.updatestatus.locks;
-
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.pentaho.di.core.util.Assert;
-
-/**
- * Test class to test the functionality of the local file locking.
- *
- * @author Administrator
- */
-public class LocalFileLockTest {
-
- /**
- * @throws java.lang.Exception
- */
- @Before public void setUp() throws Exception {
- }
-
- /**
- * @throws java.lang.Exception
- */
- @After public void tearDown() throws Exception {
- }
-
- @Test public void testingLocalFileLockingByAcquiring2Locks() {
-
- CarbonTableIdentifier carbonTableIdentifier = new CarbonTableIdentifier("databaseName", "tableName", "tableId");
- LocalFileLock localLock1 =
- new LocalFileLock(carbonTableIdentifier,
- LockUsage.METADATA_LOCK);
- Assert.assertTrue(localLock1.lock());
- LocalFileLock localLock2 =
- new LocalFileLock(carbonTableIdentifier,
- LockUsage.METADATA_LOCK);
- Assert.assertTrue(!localLock2.lock());
-
- Assert.assertTrue(localLock1.unlock());
- Assert.assertTrue(localLock2.lock());
-
- }
-
-}
[47/57] [abbrv] incubator-carbondata git commit: fix style
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
index 4907dd9..b527f37 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
@@ -20,8 +20,8 @@ package org.apache.carbondata.core.datastore.impl.btree;
import java.io.IOException;
-import org.apache.carbondata.core.datastorage.FileHolder;
import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
index 3176b9b..7035fa1 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
@@ -22,8 +22,8 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.FileHolder;
import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
index 4951c95..cfa0d2c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
@@ -20,8 +20,8 @@ package org.apache.carbondata.core.datastore.impl.btree;
import java.io.IOException;
-import org.apache.carbondata.core.datastorage.FileHolder;
import org.apache.carbondata.core.datastore.BTreeBuilderInfo;
+import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.reader.CarbonDataReaderFactory;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
new file mode 100644
index 0000000..d74cf92
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.impl.data.compressed;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.NodeMeasureDataStore;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastore.dataholder.CarbonWriteDataHolder;
+import org.apache.carbondata.core.util.ValueCompressionUtil;
+
+public abstract class AbstractHeavyCompressedDoubleArrayDataStore
+ implements NodeMeasureDataStore //NodeMeasureDataStore<double[]>
+{
+
+ private LogService LOGGER =
+ LogServiceFactory.getLogService(AbstractHeavyCompressedDoubleArrayDataStore.class.getName());
+
+ /**
+ * values.
+ */
+ protected ValueCompressionHolder[] values;
+
+ /**
+ * compressionModel.
+ */
+ protected WriterCompressModel compressionModel;
+
+ /**
+ * type
+ */
+ private char[] type;
+
+ /**
+ * AbstractHeavyCompressedDoubleArrayDataStore constructor.
+ *
+ * @param compressionModel
+ */
+ public AbstractHeavyCompressedDoubleArrayDataStore(WriterCompressModel compressionModel) {
+ this.compressionModel = compressionModel;
+ if (null != compressionModel) {
+ this.type = compressionModel.getType();
+ values =
+ new ValueCompressionHolder[compressionModel.getValueCompressionHolder().length];
+ }
+ }
+
+ // this method first invokes encoding routine to encode the data chunk,
+ // followed by invoking compression routine for preparing the data chunk for writing.
+ @Override public byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] dataHolder) {
+ byte[][] returnValue = new byte[values.length][];
+ for (int i = 0; i < compressionModel.getValueCompressionHolder().length; i++) {
+ values[i] = compressionModel.getValueCompressionHolder()[i];
+ if (type[i] != CarbonCommonConstants.BYTE_VALUE_MEASURE) {
+ // first perform encoding of the data chunk
+ values[i].setValue(
+ ValueCompressionUtil.getValueCompressor(compressionModel.getCompressionFinders()[i])
+ .getCompressedValues(compressionModel.getCompressionFinders()[i], dataHolder[i],
+ compressionModel.getMaxValue()[i],
+ compressionModel.getMantissa()[i]));
+ } else {
+ values[i].setValue(dataHolder[i].getWritableByteArrayValues());
+ }
+ values[i].compress();
+ returnValue[i] = values[i].getCompressedData();
+ }
+
+ return returnValue;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastore/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
new file mode 100644
index 0000000..793fc92
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/data/compressed/HeavyCompressedDoubleArrayDataInMemoryStore.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastore.impl.data.compressed;
+
+import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
+
+public class HeavyCompressedDoubleArrayDataInMemoryStore
+ extends AbstractHeavyCompressedDoubleArrayDataStore {
+
+ public HeavyCompressedDoubleArrayDataInMemoryStore(WriterCompressModel compressionModel) {
+ super(compressionModel);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactory.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactory.java
index 7db989a..44941c9 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/directdictionary/DirectDictionaryKeyGeneratorFactory.java
@@ -19,10 +19,8 @@
package org.apache.carbondata.core.keygenerator.directdictionary;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.keygenerator.directdictionary.timestamp
- .DateDirectDictionaryGenerator;
-import org.apache.carbondata.core.keygenerator.directdictionary.timestamp
- .TimeStampDirectDictionaryGenerator;
+import org.apache.carbondata.core.keygenerator.directdictionary.timestamp.DateDirectDictionaryGenerator;
+import org.apache.carbondata.core.keygenerator.directdictionary.timestamp.TimeStampDirectDictionaryGenerator;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.util.CarbonProperties;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java b/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
index 59aaf53..d1df15d 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/AbsoluteTableIdentifier.java
@@ -22,7 +22,7 @@ import java.io.File;
import java.io.Serializable;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.util.CarbonUtil;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfoColumnar.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfoColumnar.java b/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfoColumnar.java
index 1f3090d..46ef5ec 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfoColumnar.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/BlockletInfoColumnar.java
@@ -21,7 +21,7 @@ package org.apache.carbondata.core.metadata;
import java.util.BitSet;
-import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
public class BlockletInfoColumnar {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/metadata/ColumnIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/ColumnIdentifier.java b/core/src/main/java/org/apache/carbondata/core/metadata/ColumnIdentifier.java
new file mode 100644
index 0000000..711339b
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/ColumnIdentifier.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.metadata;
+
+import java.io.Serializable;
+import java.util.Map;
+
+/**
+ * Column unique identifier
+ */
+public class ColumnIdentifier implements Serializable {
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * column id
+ */
+ private String columnId;
+
+ /**
+ * column properties
+ */
+ private Map<String, String> columnProperties;
+
+ private DataType dataType;
+
+ /**
+ * @param columnId
+ * @param columnProperties
+ */
+ public ColumnIdentifier(String columnId, Map<String, String> columnProperties,
+ DataType dataType) {
+ this.columnId = columnId;
+ this.columnProperties = columnProperties;
+ this.dataType = dataType;
+ }
+
+ /**
+ * @return columnId
+ */
+ public String getColumnId() {
+ return columnId;
+ }
+
+ /**
+ * @param columnProperty
+ * @return
+ */
+ public String getColumnProperty(String columnProperty) {
+ String property = null;
+ if (null != columnProperties) {
+ property = columnProperties.get(columnProperty);
+ }
+ return property;
+ }
+
+ public DataType getDataType() {
+ return this.dataType;
+ }
+
+ @Override public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((columnId == null) ? 0 : columnId.hashCode());
+ return result;
+ }
+
+ @Override public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ ColumnIdentifier other = (ColumnIdentifier) obj;
+ if (columnId == null) {
+ if (other.columnId != null) {
+ return false;
+ }
+ } else if (!columnId.equals(other.columnId)) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override public String toString() {
+ return "ColumnIdentifier [columnId=" + columnId + "]";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/path/CarbonTablePath.java
index 7fb77c1..01000e8 100644
--- a/core/src/main/java/org/apache/carbondata/core/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/path/CarbonTablePath.java
@@ -21,14 +21,13 @@ package org.apache.carbondata.core.path;
import java.io.File;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.hadoop.fs.Path;
-import static org.apache.carbondata.core.constants.CarbonCommonConstants.INVALID_SEGMENT_ID;
/**
* Helps to get Table content paths.
@@ -540,9 +539,9 @@ public class CarbonTablePath extends Path {
return segmentDirSplits[1];
}
} catch (Exception e) {
- return INVALID_SEGMENT_ID;
+ return CarbonCommonConstants.INVALID_SEGMENT_ID;
}
- return INVALID_SEGMENT_ID;
+ return CarbonCommonConstants.INVALID_SEGMENT_ID;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java
index c313080..295cdc1 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java
@@ -30,7 +30,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.atomic.AtomicFileOperations;
import org.apache.carbondata.core.atomic.AtomicFileOperationsImpl;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.update.DeleteDeltaBlockDetails;
import org.apache.carbondata.core.util.CarbonUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteFilesDataReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteFilesDataReader.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteFilesDataReader.java
index 4d3c9cc..df92a3b 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteFilesDataReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteFilesDataReader.java
@@ -33,7 +33,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.update.DeleteDeltaBlockDetails;
import org.apache.carbondata.core.update.DeleteDeltaBlockletDetails;
import org.apache.carbondata.core.util.CarbonProperties;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/reader/ThriftReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/ThriftReader.java b/core/src/main/java/org/apache/carbondata/core/reader/ThriftReader.java
index 8a7abd6..0e3a3e0 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/ThriftReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/ThriftReader.java
@@ -22,7 +22,7 @@ package org.apache.carbondata.core.reader;
import java.io.DataInputStream;
import java.io.IOException;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.thrift.TBase;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
index 4472f4d..539cb6c 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImpl.java
@@ -23,7 +23,7 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.path.CarbonTablePath;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
index 967ee5c..401c148 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/ConditionalFilterResolverImpl.java
@@ -36,8 +36,7 @@ import org.apache.carbondata.core.scan.filter.FilterUtil;
import org.apache.carbondata.core.scan.filter.intf.FilterExecuterType;
import org.apache.carbondata.core.scan.filter.resolver.metadata.FilterResolverMetadata;
import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.visitor
- .FilterInfoTypeVisitorFactory;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.visitor.FilterInfoTypeVisitorFactory;
public class ConditionalFilterResolverImpl implements FilterResolverIntf {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
index fcfd924..af73b69 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/AbstractDataBlockIterator.java
@@ -24,8 +24,8 @@ import java.util.List;
import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.FileHolder;
import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.scan.collector.ScannedResultCollector;
import org.apache.carbondata.core.scan.collector.impl.DictionaryBasedResultCollector;
import org.apache.carbondata.core.scan.collector.impl.DictionaryBasedVectorResultCollector;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
index 788488a..075ad6f 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/BlocksChunkHolder.java
@@ -18,8 +18,8 @@
*/
package org.apache.carbondata.core.scan.processor;
-import org.apache.carbondata.core.datastorage.FileHolder;
import org.apache.carbondata.core.datastore.DataRefNode;
+import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java
index 6df1b98..48e3a0e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/processor/impl/DataBlockIteratorImpl.java
@@ -21,7 +21,7 @@ package org.apache.carbondata.core.scan.processor.impl;
import java.util.ArrayList;
import java.util.List;
-import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.processor.AbstractDataBlockIterator;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnarBatch;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
index 69bd273..0ae02e1 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/iterator/AbstractDetailQueryResultIterator.java
@@ -25,10 +25,10 @@ import org.apache.carbondata.common.CarbonIterator;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.FileHolder;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.datastore.DataRefNodeFinder;
+import org.apache.carbondata.core.datastore.FileHolder;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.datastore.impl.btree.BTreeDataRefNodeFinder;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
import org.apache.carbondata.core.scan.model.QueryModel;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
index 3d17985..6787851 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/scanner/impl/FilterScanner.java
@@ -23,7 +23,7 @@ import java.io.IOException;
import java.util.BitSet;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImpl.java b/core/src/main/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImpl.java
index d41c57d..1070767 100644
--- a/core/src/main/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/stats/DriverQueryStatisticsRecorderImpl.java
@@ -26,10 +26,10 @@ import java.util.concurrent.ConcurrentHashMap;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import static org.apache.carbondata.core.util.CarbonUtil.printLine;
import org.apache.commons.lang3.StringUtils;
-import static org.apache.carbondata.core.util.CarbonUtil.printLine;
/**
* Class will be used to record and log the query statistics
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java
index 3af7e65..d9d5651 100644
--- a/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java
@@ -18,11 +18,8 @@
package org.apache.carbondata.core.update;
import java.io.IOException;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -31,9 +28,9 @@ import java.util.Set;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.path.CarbonStorePath;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java
index 9ebbbfb..c43369e 100644
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java
@@ -37,7 +37,7 @@ import org.apache.carbondata.core.atomic.AtomicFileOperations;
import org.apache.carbondata.core.atomic.AtomicFileOperationsImpl;
import org.apache.carbondata.core.atomic.FileWriteOperation;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.path.CarbonStorePath;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java
index 425b039..371e6f9 100644
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java
@@ -36,9 +36,9 @@ import org.apache.carbondata.core.atomic.AtomicFileOperations;
import org.apache.carbondata.core.atomic.AtomicFileOperationsImpl;
import org.apache.carbondata.core.atomic.FileWriteOperation;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/HdfsFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/HdfsFileLock.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/HdfsFileLock.java
index ca741c1..ede5883 100644
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/HdfsFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/HdfsFileLock.java
@@ -24,8 +24,8 @@ import java.io.IOException;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.util.CarbonProperties;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLock.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLock.java
index eeca76c..1691b37 100644
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLock.java
@@ -27,7 +27,7 @@ import java.nio.channels.OverlappingFileLockException;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
index 35cca0f..c85e12d 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
@@ -31,9 +31,9 @@ import java.util.Set;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.datastore.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
import org.apache.carbondata.core.metadata.BlockletInfoColumnar;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 7e59345..19e6835 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -45,18 +45,18 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.FileHolder;
-import org.apache.carbondata.core.datastorage.columnar.ColumnGroupModel;
-import org.apache.carbondata.core.datastorage.columnar.UnBlockIndexer;
-import org.apache.carbondata.core.datastorage.compression.MeasureMetaDataModel;
-import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
+import org.apache.carbondata.core.datastore.columnar.ColumnGroupModel;
+import org.apache.carbondata.core.datastore.columnar.UnBlockIndexer;
+import org.apache.carbondata.core.datastore.compression.MeasureMetaDataModel;
+import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
index e590a5c..2615e6d 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
@@ -23,9 +23,9 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import org.apache.carbondata.core.datastorage.FileHolder;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.FileHolder;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
index 40f13f1..76c415d 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
@@ -27,31 +27,31 @@ import org.apache.carbondata.core.compression.BigDecimalCompressor;
import org.apache.carbondata.core.compression.BigIntCompressor;
import org.apache.carbondata.core.compression.DoubleCompressor;
import org.apache.carbondata.core.compression.ValueCompressor;
-import org.apache.carbondata.core.datastorage.compression.MeasureMetaDataModel;
-import org.apache.carbondata.core.datastorage.compression.ReaderCompressModel;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastorage.compression.decimal.CompressionMaxMinByte;
-import org.apache.carbondata.core.datastorage.compression.decimal.CompressionMaxMinDefault;
-import org.apache.carbondata.core.datastorage.compression.decimal.CompressionMaxMinInt;
-import org.apache.carbondata.core.datastorage.compression.decimal.CompressionMaxMinLong;
-import org.apache.carbondata.core.datastorage.compression.decimal.CompressionMaxMinShort;
-import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalByte;
-import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalDefault;
-import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalInt;
-import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalLong;
-import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalMaxMinByte;
-import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalMaxMinDefault;
-import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalMaxMinInt;
-import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalMaxMinLong;
-import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalMaxMinShort;
-import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalShort;
-import org.apache.carbondata.core.datastorage.compression.none.CompressionNoneByte;
-import org.apache.carbondata.core.datastorage.compression.none.CompressionNoneDefault;
-import org.apache.carbondata.core.datastorage.compression.none.CompressionNoneInt;
-import org.apache.carbondata.core.datastorage.compression.none.CompressionNoneLong;
-import org.apache.carbondata.core.datastorage.compression.none.CompressionNoneShort;
-import org.apache.carbondata.core.datastorage.compression.type.CompressionBigDecimal;
+import org.apache.carbondata.core.datastore.compression.MeasureMetaDataModel;
+import org.apache.carbondata.core.datastore.compression.ReaderCompressModel;
+import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastore.compression.decimal.CompressionMaxMinByte;
+import org.apache.carbondata.core.datastore.compression.decimal.CompressionMaxMinDefault;
+import org.apache.carbondata.core.datastore.compression.decimal.CompressionMaxMinInt;
+import org.apache.carbondata.core.datastore.compression.decimal.CompressionMaxMinLong;
+import org.apache.carbondata.core.datastore.compression.decimal.CompressionMaxMinShort;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalByte;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalDefault;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalInt;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalLong;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalMaxMinByte;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalMaxMinDefault;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalMaxMinInt;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalMaxMinLong;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalMaxMinShort;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalShort;
+import org.apache.carbondata.core.datastore.compression.none.CompressionNoneByte;
+import org.apache.carbondata.core.datastore.compression.none.CompressionNoneDefault;
+import org.apache.carbondata.core.datastore.compression.none.CompressionNoneInt;
+import org.apache.carbondata.core.datastore.compression.none.CompressionNoneLong;
+import org.apache.carbondata.core.datastore.compression.none.CompressionNoneShort;
+import org.apache.carbondata.core.datastore.compression.type.CompressionBigDecimal;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
public final class ValueCompressionUtil {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java
index 77fb571..51c52db 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java
@@ -27,7 +27,7 @@ import java.io.OutputStreamWriter;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.update.DeleteDeltaBlockDetails;
import org.apache.carbondata.core.util.CarbonUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
index e368788..896f334 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
@@ -28,8 +28,8 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.path.CarbonTablePath;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java b/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java
index 6a1025a..3f446f5 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java
@@ -22,7 +22,7 @@ package org.apache.carbondata.core.writer;
import java.io.DataOutputStream;
import java.io.IOException;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.hadoop.fs.FSDataOutputStream;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
index 89a1c52..5acaa49 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
@@ -26,8 +26,8 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.path.CarbonTablePath;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
index 30bfdd5..3438766 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
@@ -37,8 +37,8 @@ import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
-import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.writer.CarbonDictionaryWriter;
import org.apache.carbondata.core.writer.CarbonDictionaryWriterImpl;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/datastorage/CompressedDataMeasureWrapperTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/CompressedDataMeasureWrapperTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/CompressedDataMeasureWrapperTest.java
index 8bc6af4..b4850d0 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastorage/CompressedDataMeasureWrapperTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastorage/CompressedDataMeasureWrapperTest.java
@@ -19,8 +19,8 @@
package org.apache.carbondata.core.datastorage;
-import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder;
-import org.apache.carbondata.core.datastorage.impl.CompressedDataMeasureDataWrapper;
+import org.apache.carbondata.core.datastore.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.datastore.impl.CompressedDataMeasureDataWrapper;
import org.junit.BeforeClass;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/datastorage/DFSFileHolderImplUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/DFSFileHolderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/DFSFileHolderImplUnitTest.java
index 62700db..0b2c3bf 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastorage/DFSFileHolderImplUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastorage/DFSFileHolderImplUnitTest.java
@@ -24,7 +24,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
-import org.apache.carbondata.core.datastorage.impl.DFSFileHolderImpl;
+import org.apache.carbondata.core.datastore.impl.DFSFileHolderImpl;
import mockit.Mock;
import mockit.MockUp;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/datastorage/FileFactoryImplUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/FileFactoryImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/FileFactoryImplUnitTest.java
index 51874c0..f73994b 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastorage/FileFactoryImplUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastorage/FileFactoryImplUnitTest.java
@@ -22,7 +22,7 @@ package org.apache.carbondata.core.datastorage;
import java.io.File;
import java.io.IOException;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastore.impl.FileFactory;
import mockit.Mock;
import mockit.MockUp;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/datastorage/FileHolderImplUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/FileHolderImplUnitTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/FileHolderImplUnitTest.java
index 854b14d..c74c04e 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastorage/FileHolderImplUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastorage/FileHolderImplUnitTest.java
@@ -24,7 +24,7 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
-import org.apache.carbondata.core.datastorage.impl.FileHolderImpl;
+import org.apache.carbondata.core.datastore.impl.FileHolderImpl;
import mockit.Mock;
import mockit.MockUp;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreDataHolderUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreDataHolderUnitTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreDataHolderUnitTest.java
deleted file mode 100644
index 29d2eae..0000000
--- a/core/src/test/java/org/apache/carbondata/core/datastorage/columnar/ColumnarKeyStoreDataHolderUnitTest.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.datastorage.columnar;
-
-import java.util.List;
-
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-public class ColumnarKeyStoreDataHolderUnitTest {
-
- private static ColumnarKeyStoreDataHolder columnarKeyStoreDataHolder;
- private static ColumnarKeyStoreMetadata columnarKeyStoreMetadata;
-
- @BeforeClass public static void setup() {
- byte[] keyBlockData = new byte[] { 16, 8, 32, 40, 8, 8, 8 };
- int eachRowSize = 2;
- int[] reverseIndex = new int[] { 1, 5, 6, 3, 8 };
- columnarKeyStoreMetadata = new ColumnarKeyStoreMetadata(eachRowSize);
- columnarKeyStoreMetadata.setColumnReverseIndex(reverseIndex);
- columnarKeyStoreDataHolder =
- new ColumnarKeyStoreDataHolder(keyBlockData, columnarKeyStoreMetadata);
- }
-
- @Test public void testGetSurrogateKeyWithNullINGetColumnReverseIndex() {
- byte[] keyBlockData = new byte[] { 16, 8, 32, 40, 8, 8, 8 };
- int eachRowSize = 1;
- ColumnarKeyStoreMetadata columnarKeyStoreMetadata = new ColumnarKeyStoreMetadata(eachRowSize);
- ColumnarKeyStoreDataHolder columnarKeyStoreDataHolderNew =
- new ColumnarKeyStoreDataHolder(keyBlockData, columnarKeyStoreMetadata);
- int columnIndex = 5;
- int expected_result = 8;
- int result = columnarKeyStoreDataHolderNew.getSurrogateKey(columnIndex);
- assertEquals(expected_result, result);
- }
-
- @Test public void testGetSurrogateKeyWithNullINGetColumnReverseIndexAndRowSizeTwo() {
- byte[] keyBlockData = new byte[] { 16, 8, 32, 40, 8, 8, 8 };
- int eachRowSize = 2;
- ColumnarKeyStoreMetadata columnarKeyStoreMetadata = new ColumnarKeyStoreMetadata(eachRowSize);
- ColumnarKeyStoreDataHolder columnarKeyStoreDataHolderNew =
- new ColumnarKeyStoreDataHolder(keyBlockData, columnarKeyStoreMetadata);
- int columnIndex = 0;
- int expected_result = 4104;
- int result = columnarKeyStoreDataHolderNew.getSurrogateKey(columnIndex);
- assertEquals(expected_result, result);
- }
-
- @Test public void testGetSurrogateKeyWithNotNullINGetColumnReverseIndex() {
- int columnIndex = 0;
- int expected_result = 8232;
- int result = columnarKeyStoreDataHolder.getSurrogateKey(columnIndex);
- assertEquals(expected_result, result);
- }
-
- @Test(expected = ArrayIndexOutOfBoundsException.class)
- public void testExceptionInGetSurrogateKey() {
- int columnIndex = 10;
- int expected_result = 8232;
- int result = columnarKeyStoreDataHolder.getSurrogateKey(columnIndex);
- assertEquals(expected_result, result);
- }
-
- @Test public void testGetSurrogateKeyWithListOfByteWhileCreatingObject() {
- byte[] keyBlockData = new byte[] { 32, 64, 32, 40, 64, 8, 8 };
- List<byte[]> noDictionaryValBasedKeyBlockData = new java.util.ArrayList<>();
- noDictionaryValBasedKeyBlockData.add(keyBlockData);
- new ColumnarKeyStoreDataHolder(columnarKeyStoreMetadata);
- int columnIndex = 0;
- int expected_result = 8232;
- int result = columnarKeyStoreDataHolder.getSurrogateKey(columnIndex);
- assertEquals(expected_result, result);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/AlluxioCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/AlluxioCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/AlluxioCarbonFileTest.java
deleted file mode 100644
index 49323d8..0000000
--- a/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/AlluxioCarbonFileTest.java
+++ /dev/null
@@ -1,292 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.filesystem;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Options;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-public class AlluxioCarbonFileTest {
-
- private static AlluxioCarbonFile alluxioCarbonFile;
- private static FileStatus fileStatus;
- private static FileStatus fileStatusWithOutDirectoryPermission;
- private static String fileName;
- private static File file;
-
-
- @BeforeClass
- static public void setUp() {
- file = new File("Test.carbondata");
- if (!file.exists())
- try {
- file.createNewFile();
- } catch (IOException e) {
- e.printStackTrace();
- }
- try {
- FileOutputStream oFile = new FileOutputStream(file, true);
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- }
-
- fileStatus = new FileStatus(12L, true, 60, 120l, 180L, new Path(file.getAbsolutePath()));
- fileStatusWithOutDirectoryPermission = new FileStatus(12L, false, 60, 120l, 180L, new Path(file.getAbsolutePath()));
- fileName = file.getAbsolutePath();
- alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
- }
-
- @AfterClass
- static public void cleanUp() {
- file.delete();
- }
-
- @Test
- public void testRenameForceForException() throws IOException {
-
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- throw new IOException();
- }
-
- };
- alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
- alluxioCarbonFile.renameForce(fileName);
- }
-
- @Test
- public void testListFilesWithOutDirectoryPermission() {
- alluxioCarbonFile = new AlluxioCarbonFile(fileStatusWithOutDirectoryPermission);
- assertTrue(alluxioCarbonFile.listFiles() == null);
- }
-
- @Test
- public void testConstructorWithFilePath() {
- alluxioCarbonFile = new AlluxioCarbonFile(file.getAbsolutePath());
- assertTrue(alluxioCarbonFile instanceof AlluxioCarbonFile);
- }
-
- @Test
- public void testListFilesForNullListStatus() {
- alluxioCarbonFile = new AlluxioCarbonFile(fileStatusWithOutDirectoryPermission);
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public FileStatus[] listStatus(Path var1) throws IOException {
-
- return null;
- }
-
- };
- alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
- assertTrue(alluxioCarbonFile.listFiles().length == 0);
- }
-
- @Test
- public void testListDirectory() {
- alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public FileStatus[] listStatus(Path var1) throws IOException {
-
- FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
- }
-
- };
-
- assertTrue(alluxioCarbonFile.listFiles().length == 1);
- }
-
- @Test
- public void testListFilesForException() throws IOException {
- alluxioCarbonFile = new AlluxioCarbonFile(fileStatusWithOutDirectoryPermission);
-
- new MockUp<FileStatus>() {
- @Mock
- public Path getPath() {
- return new Path(file.getAbsolutePath());
- }
-
- };
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- throw new IOException();
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public FileStatus[] listStatus(Path var1) throws IOException {
-
- throw new IOException();
- }
-
- };
- alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
- alluxioCarbonFile.listFiles();
- }
-
- @Test
- public void testListFilesWithCarbonFilter() {
- CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
-
- @Override
- public boolean accept(CarbonFile file) {
- return true;
- }
- };
- alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
- assertTrue(alluxioCarbonFile.listFiles(carbonFileFilter).length == 1);
- }
-
- @Test
- public void testlistFilesWithoutFilter() {
- CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
-
- @Override
- public boolean accept(CarbonFile file) {
- return false;
- }
- };
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public FileStatus[] listStatus(Path var1) throws IOException {
-
- FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
- }
-
- };
- alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
- assertTrue(alluxioCarbonFile.listFiles(carbonFileFilter).length == 0);
- }
-
- @Test
- public void testGetParentFile() {
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
- new MockUp<Path>() {
- @Mock
- public Path getParent() {
- return new Path(file.getAbsolutePath()
- );
- }
-
- };
- new MockUp<FileStatus>() {
- @Mock
- public Path getPath() {
- return new Path(file.getAbsolutePath());
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public FileStatus getFileStatus(Path path) throws IOException {
-
- return new FileStatus(12L, true, 60, 120l, 180L, new Path(file.getAbsolutePath()));
- }
-
- };
-
- alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
- assertFalse(alluxioCarbonFile.getParentFile().equals(null));
- }
-
- @Test
- public void testForNonDisributedSystem() {
- alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new WebHdfsFileSystem();
- }
-
- };
- assertFalse(alluxioCarbonFile.renameForce(fileName));
- }
-
- @Test
- public void testrenameForceForDisributedSystem() {
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public void rename(Path src, Path dst, final Options.Rename... options) throws IOException {
-
- }
-
- };
-
- alluxioCarbonFile = new AlluxioCarbonFile(fileStatus);
- assertTrue(alluxioCarbonFile.renameForce(fileName));
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/HDFSCarbonFileTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/HDFSCarbonFileTest.java b/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/HDFSCarbonFileTest.java
deleted file mode 100644
index b09e14e..0000000
--- a/core/src/test/java/org/apache/carbondata/core/datastorage/filesystem/HDFSCarbonFileTest.java
+++ /dev/null
@@ -1,400 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.filesystem;
-
-import mockit.Mock;
-import mockit.MockUp;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Options;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
-import org.apache.hadoop.util.Progressable;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.BufferedWriter;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.OutputStreamWriter;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-
-public class HDFSCarbonFileTest {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(HDFSCarbonFile.class.getName());
- private static HDFSCarbonFile hdfsCarbonFile;
- private static FileStatus fileStatus = null;
- private static FileStatus fileStatusWithOutDirectoryPermission;
- private static String fileName = null;
- private static FileSystem fs = null;
- private static Path pt;
-
- @BeforeClass
- static public void setUp() throws IOException {
- Configuration config = new Configuration();
-//adding local hadoop configuration
- config.addResource(new Path("core-site.xml"));
- config.addResource(new Path("hdfs-site.xml"));
- fileName = "Test.carbondata"; //this path is HDFS path
- pt = new Path(fileName);
- fs = FileSystem.get(new Configuration(config));
- fs.create(pt);
- if (fs.exists(pt)) {
- OutputStream os = fs.create(pt,
- new Progressable() {
- public void progress() {
- LOGGER.info("Started Writing to File===");
- }
- });
- BufferedWriter br = new BufferedWriter(new OutputStreamWriter(os, "UTF-8"));
- br.write("Hello World");
- br.close();
- fs.close();
-
- fileStatus = new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName));
- fileStatusWithOutDirectoryPermission = new FileStatus(12L, false, 60, 120l, 180L, new Path(fileName));
- hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
-
- }
- }
-
- @AfterClass
- static public void cleanUp() {
- try {
- fs.delete(pt, true);
- } catch (IOException e) {
- LOGGER.error("Exception Occured" + e.getMessage());
- }
- }
-
- @Test
- public void testRenameForceForException() throws IOException {
-
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- throw new IOException();
- }
-
- };
- hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
- hdfsCarbonFile.renameForce(fileName);
- }
-
- @Test
- public void testListFilesWithOutDirectoryPermission() {
- hdfsCarbonFile = new HDFSCarbonFile(fileStatusWithOutDirectoryPermission);
- new MockUp<FileStatus>() {
- @Mock
- public boolean isDirectory() {
- return false;
- }
-
- };
-
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
-
- new MockUp<FileStatus>() {
- @Mock
- public Path getPath() {
- return new Path(fileName);
- }
-
- };
- assertEquals(hdfsCarbonFile.listFiles(), null);
- }
-
- @Test
- public void testConstructorWithFilePath() {
- hdfsCarbonFile = new HDFSCarbonFile(fileName);
- assertTrue(hdfsCarbonFile instanceof HDFSCarbonFile);
- }
-
- @Test
- public void testListFilesForNullListStatus() {
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public FileStatus[] listStatus(Path var1) throws IOException {
-
- return null;
- }
-
- };
- hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
- assertEquals(hdfsCarbonFile.listFiles().length, 0);
- }
-
- @Test
- public void testListDirectory() {
- hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public FileStatus[] listStatus(Path var1) throws IOException {
-
- FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
- }
-
- };
-
- assertEquals(hdfsCarbonFile.listFiles().length, 1);
- }
-
- @Test
- public void testListFilesForException() throws IOException {
- new HDFSCarbonFile(fileStatusWithOutDirectoryPermission);
-
- new MockUp<FileStatus>() {
- @Mock
- public Path getPath() {
- return new Path(fileName);
- }
-
- };
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- throw new IOException();
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public FileStatus[] listStatus(Path var1) throws IOException {
-
- throw new IOException();
- }
-
- };
- hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
- hdfsCarbonFile.listFiles();
- }
-
- @Test
- public void testListFilesWithCarbonFilter() {
- CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
-
- @Override
- public boolean accept(CarbonFile file) {
- return true;
- }
- };
- new MockUp<FileStatus>() {
- @Mock
- public boolean isDirectory() {
- return true;
- }
-
- };
-
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
-
- new MockUp<FileStatus>() {
- @Mock
- public Path getPath() {
- return new Path(fileName);
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public FileStatus[] listStatus(Path var1) throws IOException {
-
- FileStatus fileStatus[] = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
- }
-
- };
- hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
- assertEquals(hdfsCarbonFile.listFiles(carbonFileFilter).length, 1);
- }
-
- @Test
- public void testlistFilesWithoutFilter() {
- CarbonFileFilter carbonFileFilter = new CarbonFileFilter() {
-
- @Override
- public boolean accept(CarbonFile file) {
- return false;
- }
- };
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public FileStatus[] listStatus(Path var1) throws IOException {
-
- FileStatus[] fileStatus = new FileStatus[]{new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName))};
- return fileStatus;
- }
-
- };
- hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
- assertEquals(hdfsCarbonFile.listFiles(carbonFileFilter).length, 0);
- }
-
- @Test
- public void testGetParentFileForNull() {
-
- new MockUp<Path>() {
- @Mock
- public Path getParent() {
- return null;
- }
-
- };
- new MockUp<FileStatus>() {
- @Mock
- public Path getPath() {
- return new Path(fileName);
- }
-
- };
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
-
- new MockUp<FileStatus>() {
- @Mock
- public Path getPath() {
- return new Path(fileName);
- }
-
- };
- hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
- assertEquals(hdfsCarbonFile.getParentFile(), null);
- }
-
- @Test
- public void testGetParentFile() {
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
- new MockUp<Path>() {
- @Mock
- public Path getParent() {
- return new Path(fileName);
- }
-
- };
- new MockUp<FileStatus>() {
- @Mock
- public Path getPath() {
- return new Path(fileName);
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public FileStatus getFileStatus(Path file) throws IOException {
-
- return new FileStatus(12L, true, 60, 120l, 180L, new Path(fileName));
- }
-
- };
-
- hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
- assertTrue(hdfsCarbonFile.getParentFile() instanceof CarbonFile);
- }
-
- @Test
- public void testForNonDisributedSystem() {
- new HDFSCarbonFile(fileStatus);
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new WebHdfsFileSystem();
- }
-
- };
- assertEquals(hdfsCarbonFile.renameForce(fileName), false);
- }
-
- @Test
- public void testrenameForceForDisributedSystem() {
- new MockUp<Path>() {
- @Mock
- public FileSystem getFileSystem(Configuration conf) throws IOException {
- return new DistributedFileSystem();
- }
-
- };
- new MockUp<DistributedFileSystem>() {
- @Mock
- public void rename(Path src, Path dst, final Options.Rename... options) throws IOException {
-
- }
-
- };
- hdfsCarbonFile = new HDFSCarbonFile(fileStatus);
- assertEquals(hdfsCarbonFile.renameForce(fileName), true);
-
- }
-}
-
[19/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/update/data/BlockletLevelDeleteDeltaDataCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/update/data/BlockletLevelDeleteDeltaDataCache.java b/core/src/main/java/org/apache/carbondata/core/update/data/BlockletLevelDeleteDeltaDataCache.java
new file mode 100644
index 0000000..5b453cc
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/update/data/BlockletLevelDeleteDeltaDataCache.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.update.data;
+
+import org.roaringbitmap.RoaringBitmap;
+
+/**
+ * This class maintains delete delta data cache of each blocklet along with the block timestamp
+ */
+public class BlockletLevelDeleteDeltaDataCache {
+ private RoaringBitmap deleteDelataDataCache;
+ private String timeStamp;
+
+ public BlockletLevelDeleteDeltaDataCache(int[] deleteDeltaFileData, String timeStamp) {
+ deleteDelataDataCache = RoaringBitmap.bitmapOf(deleteDeltaFileData);
+ this.timeStamp=timeStamp;
+ }
+
+ public boolean contains(int key) {
+ return deleteDelataDataCache.contains(key);
+ }
+
+ public int getSize() {
+ return deleteDelataDataCache.getCardinality();
+ }
+
+ public String getCacheTimeStamp() {
+ return timeStamp;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/update/data/DeleteDeltaCacheLoaderIntf.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/update/data/DeleteDeltaCacheLoaderIntf.java b/core/src/main/java/org/apache/carbondata/core/update/data/DeleteDeltaCacheLoaderIntf.java
new file mode 100644
index 0000000..813d843
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/update/data/DeleteDeltaCacheLoaderIntf.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.update.data;
+
+/**
+ * This interface holds all methods required to load delete delta file data to cache
+ */
+public interface DeleteDeltaCacheLoaderIntf {
+
+ void loadDeleteDeltaFileDataToCache();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/update/data/RowCountDetailsVO.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/update/data/RowCountDetailsVO.java b/core/src/main/java/org/apache/carbondata/core/update/data/RowCountDetailsVO.java
new file mode 100644
index 0000000..15e33ad
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/update/data/RowCountDetailsVO.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.update.data;
+
+import java.io.Serializable;
+
+/**
+ * VO class Details for block.
+ */
+public class RowCountDetailsVO implements Serializable {
+
+ private static final long serialVersionUID = 1206104914918491749L;
+
+ private long totalNumberOfRows;
+
+ private long deletedRowsInBlock;
+
+ public RowCountDetailsVO(long totalNumberOfRows, long deletedRowsInBlock) {
+ this.totalNumberOfRows = totalNumberOfRows;
+ this.deletedRowsInBlock = deletedRowsInBlock;
+ }
+
+ public long getTotalNumberOfRows() {
+ return totalNumberOfRows;
+ }
+
+ public long getDeletedRowsInBlock() {
+ return deletedRowsInBlock;
+ }
+
+ @Override public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+
+ RowCountDetailsVO that = (RowCountDetailsVO) obj;
+
+ if (totalNumberOfRows != that.totalNumberOfRows) {
+ return false;
+ }
+ return deletedRowsInBlock == that.deletedRowsInBlock;
+
+ }
+
+ @Override public int hashCode() {
+ int result = (int) (totalNumberOfRows ^ (totalNumberOfRows >>> 32));
+ result = 31 * result + (int) (deletedRowsInBlock ^ (deletedRowsInBlock >>> 32));
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/update/data/UpdateVO.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/update/data/UpdateVO.java b/core/src/main/java/org/apache/carbondata/core/update/data/UpdateVO.java
new file mode 100644
index 0000000..d4f01b2
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/update/data/UpdateVO.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.update.data;
+
+import java.io.Serializable;
+
+/**
+ * VO class for storing details related to Update operation.
+ */
+public class UpdateVO implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private Long factTimestamp;
+
+ private Long updateDeltaStartTimestamp;
+
+ private String segmentId;
+
+ public Long getLatestUpdateTimestamp() {
+ return latestUpdateTimestamp;
+ }
+
+ public void setLatestUpdateTimestamp(Long latestUpdateTimestamp) {
+ this.latestUpdateTimestamp = latestUpdateTimestamp;
+ }
+
+ private Long latestUpdateTimestamp;
+
+ public Long getFactTimestamp() {
+ return factTimestamp;
+ }
+
+ public void setFactTimestamp(Long factTimestamp) {
+ this.factTimestamp = factTimestamp;
+ }
+
+ public Long getUpdateDeltaStartTimestamp() {
+ return updateDeltaStartTimestamp;
+ }
+
+ public void setUpdateDeltaStartTimestamp(Long updateDeltaStartTimestamp) {
+ this.updateDeltaStartTimestamp = updateDeltaStartTimestamp;
+ }
+
+ @Override public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ UpdateVO updateVO = (UpdateVO) o;
+ if (factTimestamp != null ?
+ !factTimestamp.equals(updateVO.factTimestamp) :
+ updateVO.factTimestamp != null) {
+ return false;
+ }
+ if (updateDeltaStartTimestamp != null ?
+ !updateDeltaStartTimestamp.equals(updateVO.updateDeltaStartTimestamp) :
+ updateVO.updateDeltaStartTimestamp != null) {
+ return false;
+ }
+ return latestUpdateTimestamp != null ?
+ latestUpdateTimestamp.equals(updateVO.latestUpdateTimestamp) :
+ updateVO.latestUpdateTimestamp == null;
+
+ }
+
+ @Override public int hashCode() {
+ int result = factTimestamp != null ? factTimestamp.hashCode() : 0;
+ result = 31 * result + (updateDeltaStartTimestamp != null ?
+ updateDeltaStartTimestamp.hashCode() :
+ 0);
+ result = 31 * result + (latestUpdateTimestamp != null ? latestUpdateTimestamp.hashCode() : 0);
+ return result;
+ }
+
+ /**
+ * This will return the update timestamp if its present or it will return the fact timestamp.
+ * @return
+ */
+ public Long getCreatedOrUpdatedTimeStamp() {
+ if (null == latestUpdateTimestamp) {
+ return factTimestamp;
+ }
+ return latestUpdateTimestamp;
+ }
+
+ public String getSegmentId() {
+ return segmentId;
+ }
+
+ public void setSegmentId(String segmentId) {
+ this.segmentId = segmentId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java
index 436e9b5..02c0860 100644
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java
@@ -31,7 +31,7 @@ import org.apache.carbondata.core.CarbonTableIdentifier;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.load.LoadMetadataDetails;
import org.apache.carbondata.core.update.CarbonUpdateUtil;
import org.apache.carbondata.core.util.CarbonUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java
index 5523925..c283012 100644
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java
@@ -29,15 +29,15 @@ import org.apache.carbondata.core.AbsoluteTableIdentifier;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.load.LoadMetadataDetails;
import org.apache.carbondata.core.reader.CarbonDeleteFilesDataReader;
import org.apache.carbondata.core.update.CarbonUpdateUtil;
import org.apache.carbondata.core.update.SegmentUpdateDetails;
import org.apache.carbondata.core.update.TupleIdEnum;
-import org.apache.carbondata.core.update.UpdateVO;
+import org.apache.carbondata.core.update.data.UpdateVO;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.fileoperations.AtomicFileOperations;
import org.apache.carbondata.fileoperations.AtomicFileOperationsImpl;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
index e43af48..360fd6f 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
@@ -34,8 +34,8 @@ import org.apache.carbondata.core.ColumnarFormatVersion;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.metadata.index.BlockIndexInfo;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
+import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
import org.apache.carbondata.core.metadata.BlockletInfoColumnar;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.format.BlockIndex;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 04a863d..2a4fb1b 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -40,7 +40,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import org.apache.carbondata.common.factory.CarbonCommonFactory;
+import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.AbsoluteTableIdentifier;
@@ -54,13 +54,13 @@ import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataC
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.core.datastorage.store.columnar.ColumnGroupModel;
-import org.apache.carbondata.core.datastorage.store.columnar.UnBlockIndexer;
-import org.apache.carbondata.core.datastorage.store.compression.MeasureMetaDataModel;
-import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.datastorage.columnar.ColumnGroupModel;
+import org.apache.carbondata.core.datastorage.columnar.UnBlockIndexer;
+import org.apache.carbondata.core.datastorage.compression.MeasureMetaDataModel;
+import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.metadata.Encoding;
@@ -72,10 +72,10 @@ import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.reader.ThriftReader;
import org.apache.carbondata.core.reader.ThriftReader.TBaseCreator;
import org.apache.carbondata.core.service.PathService;
-import org.apache.carbondata.core.update.UpdateVO;
+import org.apache.carbondata.core.update.data.UpdateVO;
import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager;
import org.apache.carbondata.format.DataChunk2;
-import org.apache.carbondata.scan.model.QueryDimension;
+import org.apache.carbondata.core.scan.model.QueryDimension;
import org.apache.commons.lang.ArrayUtils;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
index 8c5c83f..4d9e5b8 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter.java
@@ -30,8 +30,8 @@ import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk;
import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.reader.CarbonFooterReader;
import org.apache.carbondata.format.FileFooter;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
index a9be95c..40f13f1 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
@@ -27,14 +27,31 @@ import org.apache.carbondata.core.compression.BigDecimalCompressor;
import org.apache.carbondata.core.compression.BigIntCompressor;
import org.apache.carbondata.core.compression.DoubleCompressor;
import org.apache.carbondata.core.compression.ValueCompressor;
-import org.apache.carbondata.core.datastorage.store.compression.MeasureMetaDataModel;
-import org.apache.carbondata.core.datastorage.store.compression.ReaderCompressModel;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastorage.store.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastorage.store.compression.decimal.*;
-import org.apache.carbondata.core.datastorage.store.compression.nondecimal.*;
-import org.apache.carbondata.core.datastorage.store.compression.none.*;
-import org.apache.carbondata.core.datastorage.store.compression.type.*;
+import org.apache.carbondata.core.datastorage.compression.MeasureMetaDataModel;
+import org.apache.carbondata.core.datastorage.compression.ReaderCompressModel;
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
+import org.apache.carbondata.core.datastorage.compression.decimal.CompressionMaxMinByte;
+import org.apache.carbondata.core.datastorage.compression.decimal.CompressionMaxMinDefault;
+import org.apache.carbondata.core.datastorage.compression.decimal.CompressionMaxMinInt;
+import org.apache.carbondata.core.datastorage.compression.decimal.CompressionMaxMinLong;
+import org.apache.carbondata.core.datastorage.compression.decimal.CompressionMaxMinShort;
+import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalByte;
+import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalDefault;
+import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalInt;
+import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalLong;
+import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalMaxMinByte;
+import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalMaxMinDefault;
+import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalMaxMinInt;
+import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalMaxMinLong;
+import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalMaxMinShort;
+import org.apache.carbondata.core.datastorage.compression.nondecimal.CompressionNonDecimalShort;
+import org.apache.carbondata.core.datastorage.compression.none.CompressionNoneByte;
+import org.apache.carbondata.core.datastorage.compression.none.CompressionNoneDefault;
+import org.apache.carbondata.core.datastorage.compression.none.CompressionNoneInt;
+import org.apache.carbondata.core.datastorage.compression.none.CompressionNoneLong;
+import org.apache.carbondata.core.datastorage.compression.none.CompressionNoneShort;
+import org.apache.carbondata.core.datastorage.compression.type.CompressionBigDecimal;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
public final class ValueCompressionUtil {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java
index 1683f24..77fb571 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java
@@ -27,7 +27,7 @@ import java.io.OutputStreamWriter;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.update.DeleteDeltaBlockDetails;
import org.apache.carbondata.core.util.CarbonUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
index 78d5e28..9fe894a 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImpl.java
@@ -25,15 +25,15 @@ import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
-import org.apache.carbondata.common.factory.CarbonCommonFactory;
+import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.CarbonTableIdentifier;
import org.apache.carbondata.core.ColumnIdentifier;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk;
import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader;
import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReaderImpl;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java b/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java
index 93b17f3..6a1025a 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/ThriftWriter.java
@@ -22,7 +22,7 @@ package org.apache.carbondata.core.writer;
import java.io.DataOutputStream;
import java.io.IOException;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.hadoop.fs.FSDataOutputStream;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
index b6625f4..9f9d7ab 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
@@ -23,15 +23,15 @@ import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
-import org.apache.carbondata.common.factory.CarbonCommonFactory;
+import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.CarbonTableIdentifier;
import org.apache.carbondata.core.ColumnIdentifier;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.service.PathService;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/fileoperations/AtomicFileOperationsImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/fileoperations/AtomicFileOperationsImpl.java b/core/src/main/java/org/apache/carbondata/fileoperations/AtomicFileOperationsImpl.java
index 8e4c502..1bb8ba3 100644
--- a/core/src/main/java/org/apache/carbondata/fileoperations/AtomicFileOperationsImpl.java
+++ b/core/src/main/java/org/apache/carbondata/fileoperations/AtomicFileOperationsImpl.java
@@ -24,9 +24,9 @@ import java.io.DataOutputStream;
import java.io.IOException;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory.FileType;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.impl.FileFactory.FileType;
public class AtomicFileOperationsImpl implements AtomicFileOperations {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/locks/HdfsFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/locks/HdfsFileLock.java b/core/src/main/java/org/apache/carbondata/locks/HdfsFileLock.java
index ee67b4c..d61df15 100644
--- a/core/src/main/java/org/apache/carbondata/locks/HdfsFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/locks/HdfsFileLock.java
@@ -25,8 +25,8 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.CarbonTableIdentifier;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/locks/LocalFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/locks/LocalFileLock.java b/core/src/main/java/org/apache/carbondata/locks/LocalFileLock.java
index c09a42c..e61dcd5 100644
--- a/core/src/main/java/org/apache/carbondata/locks/LocalFileLock.java
+++ b/core/src/main/java/org/apache/carbondata/locks/LocalFileLock.java
@@ -28,7 +28,7 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.CarbonTableIdentifier;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
/**
* This class handles the file locking in the local file system.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/collector/ScannedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/collector/ScannedResultCollector.java b/core/src/main/java/org/apache/carbondata/scan/collector/ScannedResultCollector.java
deleted file mode 100644
index e5324a8..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/collector/ScannedResultCollector.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.collector;
-
-import java.util.List;
-
-import org.apache.carbondata.scan.result.AbstractScannedResult;
-import org.apache.carbondata.scan.result.vector.CarbonColumnarBatch;
-
-/**
- * Interface which will be used to aggregate the scan result
- */
-public interface ScannedResultCollector {
-
- /**
- * Below method will be used to aggregate the scanned result
- *
- * @param scannedResult scanned result
- * @return how many records was aggregated
- */
- List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize);
-
- /**
- * Collects data in columnar format.
- * @param scannedResult
- * @param columnarBatch
- */
- void collectVectorBatch(AbstractScannedResult scannedResult, CarbonColumnarBatch columnarBatch);
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/collector/impl/AbstractScannedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/collector/impl/AbstractScannedResultCollector.java b/core/src/main/java/org/apache/carbondata/scan/collector/impl/AbstractScannedResultCollector.java
deleted file mode 100644
index ff4c1da..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/collector/impl/AbstractScannedResultCollector.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.collector.impl;
-
-import java.util.List;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
-import org.apache.carbondata.scan.collector.ScannedResultCollector;
-import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
-import org.apache.carbondata.scan.executor.util.QueryUtil;
-import org.apache.carbondata.scan.result.AbstractScannedResult;
-import org.apache.carbondata.scan.result.vector.CarbonColumnarBatch;
-import org.apache.carbondata.scan.wrappers.ByteArrayWrapper;
-
-/**
- * It is not a collector it is just a scanned result holder.
- */
-public abstract class AbstractScannedResultCollector implements ScannedResultCollector {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(AbstractScannedResultCollector.class.getName());
-
- /**
- * restructuring info
- */
- private KeyStructureInfo restructureInfos;
-
- /**
- * table block execution infos
- */
- protected BlockExecutionInfo tableBlockExecutionInfos;
-
- /**
- * Measure ordinals
- */
- protected int[] measuresOrdinal;
-
- /**
- * to check whether measure exists in current table block or not this to
- * handle restructuring scenario
- */
- protected boolean[] isMeasureExistsInCurrentBlock;
-
- /**
- * default value of the measures in case of restructuring some measure wont
- * be present in the table so in that default value will be used to
- * aggregate the data for that measure columns
- */
- private Object[] measureDefaultValue;
-
- /**
- * measure datatypes.
- */
- protected DataType[] measureDatatypes;
-
- public AbstractScannedResultCollector(BlockExecutionInfo blockExecutionInfos) {
- this.tableBlockExecutionInfos = blockExecutionInfos;
- restructureInfos = blockExecutionInfos.getKeyStructureInfo();
- measuresOrdinal = tableBlockExecutionInfos.getAggregatorInfo().getMeasureOrdinals();
- isMeasureExistsInCurrentBlock = tableBlockExecutionInfos.getAggregatorInfo().getMeasureExists();
- measureDefaultValue = tableBlockExecutionInfos.getAggregatorInfo().getDefaultValues();
- this.measureDatatypes = tableBlockExecutionInfos.getAggregatorInfo().getMeasureDataTypes();
- }
-
- protected void fillMeasureData(Object[] msrValues, int offset,
- AbstractScannedResult scannedResult) {
- for (short i = 0; i < measuresOrdinal.length; i++) {
- // if measure exists is block then pass measure column
- // data chunk to the collector
- if (isMeasureExistsInCurrentBlock[i]) {
- msrValues[i + offset] = getMeasureData(scannedResult.getMeasureChunk(measuresOrdinal[i]),
- scannedResult.getCurrenrRowId(), measureDatatypes[i]);
- } else {
- // if not then get the default value and use that value in aggregation
- msrValues[i + offset] = measureDefaultValue[i];
- }
- }
- }
-
- private Object getMeasureData(MeasureColumnDataChunk dataChunk, int index, DataType dataType) {
- if (!dataChunk.getNullValueIndexHolder().getBitSet().get(index)) {
- switch (dataType) {
- case SHORT:
- case INT:
- case LONG:
- return dataChunk.getMeasureDataHolder().getReadableLongValueByIndex(index);
- case DECIMAL:
- return org.apache.spark.sql.types.Decimal
- .apply(dataChunk.getMeasureDataHolder().getReadableBigDecimalValueByIndex(index));
- default:
- return dataChunk.getMeasureDataHolder().getReadableDoubleValueByIndex(index);
- }
- }
- return null;
- }
-
- /**
- * Below method will used to get the result
- */
- protected void updateData(List<Object[]> listBasedResult) {
- if (tableBlockExecutionInfos.isFixedKeyUpdateRequired()) {
- updateKeyWithLatestBlockKeygenerator(listBasedResult);
- }
- }
-
- /**
- * Below method will be used to update the fixed length key with the
- * latest block key generator
- *
- * @return updated block
- */
- private void updateKeyWithLatestBlockKeygenerator(List<Object[]> listBasedResult) {
- try {
- long[] data = null;
- ByteArrayWrapper key = null;
- for (int i = 0; i < listBasedResult.size(); i++) {
- // get the key
- key = (ByteArrayWrapper) listBasedResult.get(i)[0];
- // unpack the key with table block key generator
- data = tableBlockExecutionInfos.getBlockKeyGenerator()
- .getKeyArray(key.getDictionaryKey(), tableBlockExecutionInfos.getMaskedByteForBlock());
- // packed the key with latest block key generator
- // and generate the masked key for that key
- key.setDictionaryKey(QueryUtil
- .getMaskedKey(restructureInfos.getKeyGenerator().generateKey(data),
- restructureInfos.getMaxKey(), restructureInfos.getMaskByteRanges(),
- restructureInfos.getMaskByteRanges().length));
- }
- } catch (KeyGenException e) {
- LOGGER.error(e);
- }
- }
-
- @Override public void collectVectorBatch(AbstractScannedResult scannedResult,
- CarbonColumnarBatch columnarBatch) {
- throw new UnsupportedOperationException("Works only for batch collectors");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedResultCollector.java b/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedResultCollector.java
deleted file mode 100644
index 853bfec..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedResultCollector.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.collector.impl;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.carbondata.common.iudprocessor.cache.BlockletLevelDeleteDeltaDataCache;
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.core.metadata.Encoding;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
-import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
-import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.scan.filter.GenericQueryType;
-import org.apache.carbondata.scan.model.QueryDimension;
-import org.apache.carbondata.scan.model.QueryMeasure;
-import org.apache.carbondata.scan.result.AbstractScannedResult;
-
-import org.apache.commons.lang3.ArrayUtils;
-/**
- * It is not a collector it is just a scanned result holder.
- */
-public class DictionaryBasedResultCollector extends AbstractScannedResultCollector {
-
- public DictionaryBasedResultCollector(BlockExecutionInfo blockExecutionInfos) {
- super(blockExecutionInfos);
- }
-
- /**
- * This method will add a record both key and value to list object
- * it will keep track of how many record is processed, to handle limit scenario
- */
- @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) {
-
- List<Object[]> listBasedResult = new ArrayList<>(batchSize);
- boolean isMsrsPresent = measureDatatypes.length > 0;
-
- QueryDimension[] queryDimensions = tableBlockExecutionInfos.getQueryDimensions();
- List<Integer> dictionaryIndexes = new ArrayList<Integer>();
- for (int i = 0; i < queryDimensions.length; i++) {
- if(queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY) ||
- queryDimensions[i].getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY) ) {
- dictionaryIndexes.add(queryDimensions[i].getDimension().getOrdinal());
- }
- }
- int[] primitive = ArrayUtils.toPrimitive(dictionaryIndexes.toArray(
- new Integer[dictionaryIndexes.size()]));
- Arrays.sort(primitive);
- int[] actualIndexInSurrogateKey = new int[dictionaryIndexes.size()];
- int index = 0;
- for (int i = 0; i < queryDimensions.length; i++) {
- if(queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY) ||
- queryDimensions[i].getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY) ) {
- actualIndexInSurrogateKey[index++] = Arrays.binarySearch(primitive,
- queryDimensions[i].getDimension().getOrdinal());
- }
- }
-
- QueryMeasure[] queryMeasures = tableBlockExecutionInfos.getQueryMeasures();
- BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache =
- scannedResult.getDeleteDeltaDataCache();
- Map<Integer, GenericQueryType> comlexDimensionInfoMap =
- tableBlockExecutionInfos.getComlexDimensionInfoMap();
- boolean[] dictionaryEncodingArray = CarbonUtil.getDictionaryEncodingArray(queryDimensions);
- boolean[] directDictionaryEncodingArray =
- CarbonUtil.getDirectDictionaryEncodingArray(queryDimensions);
- boolean[] implictColumnArray = CarbonUtil.getImplicitColumnArray(queryDimensions);
- boolean[] complexDataTypeArray = CarbonUtil.getComplexDataTypeArray(queryDimensions);
- int dimSize = queryDimensions.length;
- boolean isDimensionsExist = dimSize > 0;
- int[] order = new int[dimSize + queryMeasures.length];
- for (int i = 0; i < dimSize; i++) {
- order[i] = queryDimensions[i].getQueryOrder();
- }
- for (int i = 0; i < queryMeasures.length; i++) {
- order[i + dimSize] = queryMeasures[i].getQueryOrder();
- }
- // scan the record and add to list
- int rowCounter = 0;
- int dictionaryColumnIndex = 0;
- int noDictionaryColumnIndex = 0;
- int complexTypeColumnIndex = 0;
- int[] surrogateResult;
- String[] noDictionaryKeys;
- byte[][] complexTypeKeyArray;
- while (scannedResult.hasNext() && rowCounter < batchSize) {
- Object[] row = new Object[dimSize + queryMeasures.length];
- if (isDimensionsExist) {
- surrogateResult = scannedResult.getDictionaryKeyIntegerArray();
- noDictionaryKeys = scannedResult.getNoDictionaryKeyStringArray();
- complexTypeKeyArray = scannedResult.getComplexTypeKeyArray();
- dictionaryColumnIndex = 0;
- noDictionaryColumnIndex = 0;
- complexTypeColumnIndex = 0;
- for (int i = 0; i < dimSize; i++) {
- if (!dictionaryEncodingArray[i]) {
- if (implictColumnArray[i]) {
- if (CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID
- .equals(queryDimensions[i].getDimension().getColName())) {
- row[order[i]] = DataTypeUtil.getDataBasedOnDataType(
- scannedResult.getBlockletId() + CarbonCommonConstants.FILE_SEPARATOR
- + scannedResult.getCurrenrRowId(), DataType.STRING);
- } else {
- row[order[i]] = DataTypeUtil
- .getDataBasedOnDataType(scannedResult.getBlockletId(), DataType.STRING);
- }
- } else {
- row[order[i]] = DataTypeUtil
- .getDataBasedOnDataType(noDictionaryKeys[noDictionaryColumnIndex++],
- queryDimensions[i].getDimension().getDataType());
- }
- } else if (directDictionaryEncodingArray[i]) {
- DirectDictionaryGenerator directDictionaryGenerator =
- DirectDictionaryKeyGeneratorFactory
- .getDirectDictionaryGenerator(queryDimensions[i].getDimension().getDataType());
- if (directDictionaryGenerator != null) {
- row[order[i]] = directDictionaryGenerator.getValueFromSurrogate(
- surrogateResult[actualIndexInSurrogateKey[dictionaryColumnIndex++]]);
- }
- } else if (complexDataTypeArray[i]) {
- row[order[i]] = comlexDimensionInfoMap
- .get(queryDimensions[i].getDimension().getOrdinal())
- .getDataBasedOnDataTypeFromSurrogates(
- ByteBuffer.wrap(complexTypeKeyArray[complexTypeColumnIndex++]));
- } else {
- row[order[i]] = surrogateResult[actualIndexInSurrogateKey[dictionaryColumnIndex++]];
- }
- }
-
- } else {
- scannedResult.incrementCounter();
- }
- if (null != deleteDeltaDataCache && deleteDeltaDataCache
- .contains(scannedResult.getCurrenrRowId())) {
- continue;
- }
- if (isMsrsPresent) {
- Object[] msrValues = new Object[measureDatatypes.length];
- fillMeasureData(msrValues, 0, scannedResult);
- for (int i = 0; i < msrValues.length; i++) {
- row[order[i + dimSize]] = msrValues[i];
- }
- }
- listBasedResult.add(row);
- rowCounter++;
- }
- return listBasedResult;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedVectorResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedVectorResultCollector.java b/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedVectorResultCollector.java
deleted file mode 100644
index 41247e2..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedVectorResultCollector.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.collector.impl;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.carbondata.core.metadata.Encoding;
-import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
-import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.scan.model.QueryDimension;
-import org.apache.carbondata.scan.model.QueryMeasure;
-import org.apache.carbondata.scan.result.AbstractScannedResult;
-import org.apache.carbondata.scan.result.vector.CarbonColumnarBatch;
-import org.apache.carbondata.scan.result.vector.ColumnVectorInfo;
-import org.apache.carbondata.scan.result.vector.MeasureDataVectorProcessor;
-
-/**
- * It is not a collector it is just a scanned result holder.
- */
-public class DictionaryBasedVectorResultCollector extends AbstractScannedResultCollector {
-
- private ColumnVectorInfo[] dictionaryInfo;
-
- private ColumnVectorInfo[] noDictionaryInfo;
-
- private ColumnVectorInfo[] complexInfo;
-
- private ColumnVectorInfo[] measureInfo;
-
- private ColumnVectorInfo[] allColumnInfo;
-
- public DictionaryBasedVectorResultCollector(BlockExecutionInfo blockExecutionInfos) {
- super(blockExecutionInfos);
- QueryDimension[] queryDimensions = tableBlockExecutionInfos.getQueryDimensions();
- QueryMeasure[] queryMeasures = tableBlockExecutionInfos.getQueryMeasures();
- measureInfo = new ColumnVectorInfo[queryMeasures.length];
- allColumnInfo = new ColumnVectorInfo[queryDimensions.length + queryMeasures.length];
- List<ColumnVectorInfo> dictInfoList = new ArrayList<>();
- List<ColumnVectorInfo> noDictInfoList = new ArrayList<>();
- List<ColumnVectorInfo> complexList = new ArrayList<>();
- for (int i = 0; i < queryDimensions.length; i++) {
- if (!queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY)) {
- ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo();
- noDictInfoList.add(columnVectorInfo);
- columnVectorInfo.dimension = queryDimensions[i];
- columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal();
- allColumnInfo[queryDimensions[i].getQueryOrder()] = columnVectorInfo;
- } else if (queryDimensions[i].getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
- ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo();
- dictInfoList.add(columnVectorInfo);
- columnVectorInfo.dimension = queryDimensions[i];
- columnVectorInfo.directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
- .getDirectDictionaryGenerator(queryDimensions[i].getDimension().getDataType());
- columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal();
- allColumnInfo[queryDimensions[i].getQueryOrder()] = columnVectorInfo;
- } else if (queryDimensions[i].getDimension().isComplex()) {
- ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo();
- complexList.add(columnVectorInfo);
- columnVectorInfo.dimension = queryDimensions[i];
- columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal();
- columnVectorInfo.genericQueryType =
- tableBlockExecutionInfos.getComlexDimensionInfoMap().get(columnVectorInfo.ordinal);
- allColumnInfo[queryDimensions[i].getQueryOrder()] = columnVectorInfo;
- } else {
- ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo();
- dictInfoList.add(columnVectorInfo);
- columnVectorInfo.dimension = queryDimensions[i];
- columnVectorInfo.ordinal = queryDimensions[i].getDimension().getOrdinal();
- allColumnInfo[queryDimensions[i].getQueryOrder()] = columnVectorInfo;
- }
- }
- for (int i = 0; i < queryMeasures.length; i++) {
- ColumnVectorInfo columnVectorInfo = new ColumnVectorInfo();
- columnVectorInfo.measureVectorFiller = MeasureDataVectorProcessor.MeasureVectorFillerFactory
- .getMeasureVectorFiller(queryMeasures[i].getMeasure().getDataType());
- columnVectorInfo.ordinal = queryMeasures[i].getMeasure().getOrdinal();
- columnVectorInfo.measure = queryMeasures[i];
- measureInfo[i] = columnVectorInfo;
- allColumnInfo[queryMeasures[i].getQueryOrder()] = columnVectorInfo;
- }
- dictionaryInfo = dictInfoList.toArray(new ColumnVectorInfo[dictInfoList.size()]);
- noDictionaryInfo = noDictInfoList.toArray(new ColumnVectorInfo[noDictInfoList.size()]);
- complexInfo = complexList.toArray(new ColumnVectorInfo[complexList.size()]);
- Arrays.sort(dictionaryInfo);
- Arrays.sort(noDictionaryInfo);
- Arrays.sort(complexInfo);
- }
-
- @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) {
- throw new UnsupportedOperationException("collectData is not supported here");
- }
-
- @Override public void collectVectorBatch(AbstractScannedResult scannedResult,
- CarbonColumnarBatch columnarBatch) {
- int rowCounter = scannedResult.getRowCounter();
- int availableRows = scannedResult.numberOfOutputRows() - rowCounter;
- int requiredRows = columnarBatch.getBatchSize() - columnarBatch.getActualSize();
- requiredRows = Math.min(requiredRows, availableRows);
- if (requiredRows < 1) {
- return;
- }
- for (int i = 0; i < allColumnInfo.length; i++) {
- allColumnInfo[i].size = requiredRows;
- allColumnInfo[i].offset = rowCounter;
- allColumnInfo[i].vectorOffset = columnarBatch.getRowCounter();
- allColumnInfo[i].vector = columnarBatch.columnVectors[i];
- }
-
- scannedResult.fillColumnarDictionaryBatch(dictionaryInfo);
- scannedResult.fillColumnarNoDictionaryBatch(noDictionaryInfo);
- scannedResult.fillColumnarMeasureBatch(measureInfo, measuresOrdinal);
- scannedResult.fillColumnarComplexBatch(complexInfo);
- scannedResult.setRowCounter(rowCounter + requiredRows);
- columnarBatch.setActualSize(columnarBatch.getActualSize() + requiredRows);
- columnarBatch.setRowCounter(columnarBatch.getRowCounter() + requiredRows);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollector.java b/core/src/main/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollector.java
deleted file mode 100644
index b61ef97..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollector.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.collector.impl;
-
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.common.iudprocessor.cache.BlockletLevelDeleteDeltaDataCache;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
-import org.apache.carbondata.scan.model.QueryMeasure;
-import org.apache.carbondata.scan.result.AbstractScannedResult;
-import org.apache.carbondata.scan.wrappers.ByteArrayWrapper;
-
-
-/**
- * It is not a collector it is just a scanned result holder.
- */
-public class RawBasedResultCollector extends AbstractScannedResultCollector {
-
- public RawBasedResultCollector(BlockExecutionInfo blockExecutionInfos) {
- super(blockExecutionInfos);
- }
-
- /**
- * This method will add a record both key and value to list object
- * it will keep track of how many record is processed, to handle limit scenario
- */
- @Override public List<Object[]> collectData(AbstractScannedResult scannedResult, int batchSize) {
- List<Object[]> listBasedResult = new ArrayList<>(batchSize);
- QueryMeasure[] queryMeasures = tableBlockExecutionInfos.getQueryMeasures();
- ByteArrayWrapper wrapper = null;
- BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache =
- scannedResult.getDeleteDeltaDataCache();
- // scan the record and add to list
- int rowCounter = 0;
- while (scannedResult.hasNext() && rowCounter < batchSize) {
- Object[] row = new Object[1 + queryMeasures.length];
- wrapper = new ByteArrayWrapper();
- wrapper.setDictionaryKey(scannedResult.getDictionaryKeyArray());
- wrapper.setNoDictionaryKeys(scannedResult.getNoDictionaryKeyArray());
- wrapper.setComplexTypesKeys(scannedResult.getComplexTypeKeyArray());
- wrapper.setImplicitColumnByteArray(scannedResult.getBlockletId()
- .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
- if (null != deleteDeltaDataCache && deleteDeltaDataCache
- .contains(scannedResult.getCurrenrRowId())) {
- continue;
- }
- row[0] = wrapper;
- fillMeasureData(row, 1, scannedResult);
- listBasedResult.add(row);
- rowCounter++;
- }
- updateData(listBasedResult);
- return listBasedResult;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/complextypes/ArrayQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/complextypes/ArrayQueryType.java b/core/src/main/java/org/apache/carbondata/scan/complextypes/ArrayQueryType.java
deleted file mode 100644
index b867669..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/complextypes/ArrayQueryType.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.complextypes;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.scan.filter.GenericQueryType;
-import org.apache.carbondata.scan.processor.BlocksChunkHolder;
-
-public class ArrayQueryType extends ComplexQueryType implements GenericQueryType {
-
- private GenericQueryType children;
-
- public ArrayQueryType(String name, String parentname, int blockIndex) {
- super(name, parentname, blockIndex);
- }
-
- @Override public void addChildren(GenericQueryType children) {
- if (this.getName().equals(children.getParentname())) {
- this.children = children;
- } else {
- this.children.addChildren(children);
- }
- }
-
- @Override public String getName() {
- return name;
- }
-
- @Override public void setName(String name) {
- this.name = name;
- }
-
- @Override public String getParentname() {
- return parentname;
- }
-
- @Override public void setParentname(String parentname) {
- this.parentname = parentname;
-
- }
-
- public void parseBlocksAndReturnComplexColumnByteArray(
- DimensionColumnDataChunk[] dimensionColumnDataChunks, int rowNumber,
- DataOutputStream dataOutputStream) throws IOException {
- byte[] input = new byte[8];
- copyBlockDataChunk(dimensionColumnDataChunks, rowNumber, input);
- ByteBuffer byteArray = ByteBuffer.wrap(input);
- int dataLength = byteArray.getInt();
- dataOutputStream.writeInt(dataLength);
- if (dataLength > 0) {
- int columnIndex = byteArray.getInt();
- for (int i = 0; i < dataLength; i++) {
- children
- .parseBlocksAndReturnComplexColumnByteArray(dimensionColumnDataChunks, columnIndex++,
- dataOutputStream);
- }
- }
- }
-
- @Override public int getColsCount() {
- return children.getColsCount() + 1;
- }
-
- @Override public void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder)
- throws IOException {
- readBlockDataChunk(blockChunkHolder);
- children.fillRequiredBlockData(blockChunkHolder);
- }
-
- @Override public Object getDataBasedOnDataTypeFromSurrogates(ByteBuffer surrogateData) {
- int dataLength = surrogateData.getInt();
- if (dataLength == -1) {
- return null;
- }
- Object[] data = new Object[dataLength];
- for (int i = 0; i < dataLength; i++) {
- data[i] = children.getDataBasedOnDataTypeFromSurrogates(surrogateData);
- }
- return data;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/complextypes/ComplexQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/complextypes/ComplexQueryType.java b/core/src/main/java/org/apache/carbondata/scan/complextypes/ComplexQueryType.java
deleted file mode 100644
index 2a21a6d..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/complextypes/ComplexQueryType.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.complextypes;
-
-import java.io.IOException;
-
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.scan.filter.GenericQueryType;
-import org.apache.carbondata.scan.processor.BlocksChunkHolder;
-
-public class ComplexQueryType {
- protected GenericQueryType children;
-
- protected String name;
-
- protected String parentname;
-
- protected int blockIndex;
-
- public ComplexQueryType(String name, String parentname, int blockIndex) {
- this.name = name;
- this.parentname = parentname;
- this.blockIndex = blockIndex;
- }
-
- /**
- * Method will copy the block chunk holder data to the passed
- * byte[], this method is also used by child
- *
- * @param rowNumber
- * @param input
- */
- protected void copyBlockDataChunk(DimensionColumnDataChunk[] dimensionColumnDataChunks,
- int rowNumber, byte[] input) {
- byte[] data = dimensionColumnDataChunks[blockIndex].getChunkData(rowNumber);
- System.arraycopy(data, 0, input, 0, data.length);
- }
-
- /*
- * This method will read the block data chunk from the respective block
- */
- protected void readBlockDataChunk(BlocksChunkHolder blockChunkHolder) throws IOException {
- if (null == blockChunkHolder.getDimensionDataChunk()[blockIndex]) {
- blockChunkHolder.getDimensionDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
- .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/complextypes/PrimitiveQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/complextypes/PrimitiveQueryType.java b/core/src/main/java/org/apache/carbondata/scan/complextypes/PrimitiveQueryType.java
deleted file mode 100644
index 72805f6..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/complextypes/PrimitiveQueryType.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.complextypes;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.core.cache.dictionary.Dictionary;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
-import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
-import org.apache.carbondata.core.keygenerator.mdkey.Bits;
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.core.util.DataTypeUtil;
-import org.apache.carbondata.scan.filter.GenericQueryType;
-import org.apache.carbondata.scan.processor.BlocksChunkHolder;
-
-public class PrimitiveQueryType extends ComplexQueryType implements GenericQueryType {
-
- private String name;
- private String parentname;
-
- private int keySize;
-
- private int blockIndex;
-
- private Dictionary dictionary;
-
- private DataType dataType;
-
- private boolean isDirectDictionary;
-
- public PrimitiveQueryType(String name, String parentname, int blockIndex, DataType dataType,
- int keySize, Dictionary dictionary, boolean isDirectDictionary) {
- super(name, parentname, blockIndex);
- this.dataType = dataType;
- this.keySize = keySize;
- this.dictionary = dictionary;
- this.name = name;
- this.parentname = parentname;
- this.blockIndex = blockIndex;
- this.isDirectDictionary = isDirectDictionary;
- }
-
- @Override public void addChildren(GenericQueryType children) {
-
- }
-
- @Override public String getName() {
- return name;
- }
-
- @Override public void setName(String name) {
- this.name = name;
- }
-
- @Override public String getParentname() {
- return parentname;
- }
-
- @Override public void setParentname(String parentname) {
- this.parentname = parentname;
-
- }
-
- @Override public int getColsCount() {
- return 1;
- }
-
- @Override public void parseBlocksAndReturnComplexColumnByteArray(
- DimensionColumnDataChunk[] dimensionDataChunks, int rowNumber,
- DataOutputStream dataOutputStream) throws IOException {
- byte[] currentVal =
- new byte[dimensionDataChunks[blockIndex].getColumnValueSize()];
- copyBlockDataChunk(dimensionDataChunks, rowNumber, currentVal);
- dataOutputStream.write(currentVal);
- }
-
- @Override public void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder)
- throws IOException {
- readBlockDataChunk(blockChunkHolder);
- }
-
- @Override public Object getDataBasedOnDataTypeFromSurrogates(ByteBuffer surrogateData) {
- byte[] data = new byte[keySize];
- surrogateData.get(data);
- Bits bit = new Bits(new int[]{keySize * 8});
- int surrgateValue = (int)bit.getKeyArray(data, 0)[0];
- Object actualData = null;
- if (isDirectDictionary) {
- DirectDictionaryGenerator directDictionaryGenerator = DirectDictionaryKeyGeneratorFactory
- .getDirectDictionaryGenerator(dataType);
- actualData = directDictionaryGenerator.getValueFromSurrogate(surrgateValue);
- } else {
- String dictionaryValueForKey = dictionary.getDictionaryValueForKey(surrgateValue);
- actualData = DataTypeUtil.getDataBasedOnDataType(dictionaryValueForKey, this.dataType);
- }
- return actualData;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/complextypes/StructQueryType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/complextypes/StructQueryType.java b/core/src/main/java/org/apache/carbondata/scan/complextypes/StructQueryType.java
deleted file mode 100644
index 4064efe..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/complextypes/StructQueryType.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.complextypes;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
-import org.apache.carbondata.scan.filter.GenericQueryType;
-import org.apache.carbondata.scan.processor.BlocksChunkHolder;
-
-public class StructQueryType extends ComplexQueryType implements GenericQueryType {
-
- private List<GenericQueryType> children = new ArrayList<GenericQueryType>();
- private String name;
- private String parentname;
-
- public StructQueryType(String name, String parentname, int blockIndex) {
- super(name, parentname, blockIndex);
- this.name = name;
- this.parentname = parentname;
- }
-
- @Override public void addChildren(GenericQueryType newChild) {
- if (this.getName().equals(newChild.getParentname())) {
- this.children.add(newChild);
- } else {
- for (GenericQueryType child : this.children) {
- child.addChildren(newChild);
- }
- }
-
- }
-
- @Override public String getName() {
- return name;
- }
-
- @Override public void setName(String name) {
- this.name = name;
- }
-
- @Override public String getParentname() {
- return parentname;
- }
-
- @Override public void setParentname(String parentname) {
- this.parentname = parentname;
-
- }
-
- @Override public int getColsCount() {
- int colsCount = 1;
- for (int i = 0; i < children.size(); i++) {
- colsCount += children.get(i).getColsCount();
- }
- return colsCount;
- }
-
- @Override public void parseBlocksAndReturnComplexColumnByteArray(
- DimensionColumnDataChunk[] dimensionColumnDataChunks, int rowNumber,
- DataOutputStream dataOutputStream) throws IOException {
- byte[] input = new byte[8];
- copyBlockDataChunk(dimensionColumnDataChunks, rowNumber, input);
- ByteBuffer byteArray = ByteBuffer.wrap(input);
- int childElement = byteArray.getInt();
- dataOutputStream.writeInt(childElement);
- if (childElement > 0){
- for (int i = 0; i < childElement; i++) {
- children.get(i)
- .parseBlocksAndReturnComplexColumnByteArray(dimensionColumnDataChunks, rowNumber,
- dataOutputStream);
- }
- }
- }
-
- @Override public void fillRequiredBlockData(BlocksChunkHolder blockChunkHolder)
- throws IOException {
- readBlockDataChunk(blockChunkHolder);
-
- for (int i = 0; i < children.size(); i++) {
- children.get(i).fillRequiredBlockData(blockChunkHolder);
- }
- }
-
- @Override public Object getDataBasedOnDataTypeFromSurrogates(ByteBuffer surrogateData) {
- int childLength = surrogateData.getInt();
- Object[] fields = new Object[childLength];
- for (int i = 0; i < childLength; i++) {
- fields[i] = children.get(i).getDataBasedOnDataTypeFromSurrogates(surrogateData);
- }
- return fields;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/executor/QueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/QueryExecutor.java b/core/src/main/java/org/apache/carbondata/scan/executor/QueryExecutor.java
deleted file mode 100644
index 949771c..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/executor/QueryExecutor.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.executor;
-
-import java.io.IOException;
-
-import org.apache.carbondata.common.CarbonIterator;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
-import org.apache.carbondata.scan.model.QueryModel;
-
-/**
- * Interface for carbon query executor.
- * Will be used to execute the query based on the query model
- * and will return the iterator over query result
- */
-public interface QueryExecutor<E> {
-
- /**
- * Below method will be used to execute the query based on query model passed from driver
- *
- * @param queryModel query details
- * @return query result iterator
- * @throws QueryExecutionException if any failure while executing the query
- * @throws IOException if fail to read files
- */
- CarbonIterator<E> execute(QueryModel queryModel)
- throws QueryExecutionException, IOException;
-
- /**
- * Below method will be used for cleanup
- *
- * @throws QueryExecutionException
- */
- void finish() throws QueryExecutionException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/executor/QueryExecutorFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/QueryExecutorFactory.java b/core/src/main/java/org/apache/carbondata/scan/executor/QueryExecutorFactory.java
deleted file mode 100644
index 947b992..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/executor/QueryExecutorFactory.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.executor;
-
-import org.apache.carbondata.scan.executor.impl.DetailQueryExecutor;
-import org.apache.carbondata.scan.executor.impl.VectorDetailQueryExecutor;
-import org.apache.carbondata.scan.model.QueryModel;
-
-/**
- * Factory class to get the query executor from RDD
- * This will return the executor based on query type
- */
-public class QueryExecutorFactory {
-
- public static QueryExecutor getQueryExecutor(QueryModel queryModel) {
- if (queryModel.isVectorReader()) {
- return new VectorDetailQueryExecutor();
- } else {
- return new DetailQueryExecutor();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/scan/executor/exception/QueryExecutionException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/exception/QueryExecutionException.java b/core/src/main/java/org/apache/carbondata/scan/executor/exception/QueryExecutionException.java
deleted file mode 100644
index 5634e0f..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/executor/exception/QueryExecutionException.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.scan.executor.exception;
-
-import java.util.Locale;
-
-/**
- * Exception class for query execution
- *
- * @author Administrator
- */
-public class QueryExecutionException extends Exception {
-
- /**
- * default serial version ID.
- */
- private static final long serialVersionUID = 1L;
-
- /**
- * The Error message.
- */
- private String msg = "";
-
- /**
- * Constructor
- *
- * @param msg The error message for this exception.
- */
- public QueryExecutionException(String msg) {
- super(msg);
- this.msg = msg;
- }
-
- /**
- * Constructor
- *
- * @param msg The error message for this exception.
- */
- public QueryExecutionException(String msg, Throwable t) {
- super(msg, t);
- this.msg = msg;
- }
-
- /**
- * Constructor
- *
- * @param t
- */
- public QueryExecutionException(Throwable t) {
- super(t);
- }
-
- /**
- * This method is used to get the localized message.
- *
- * @param locale - A Locale object represents a specific geographical,
- * political, or cultural region.
- * @return - Localized error message.
- */
- public String getLocalizedMessage(Locale locale) {
- return "";
- }
-
- /**
- * getLocalizedMessage
- */
- @Override public String getLocalizedMessage() {
- return super.getLocalizedMessage();
- }
-
- /**
- * getMessage
- */
- public String getMessage() {
- return this.msg;
- }
-
-}
[54/57] [abbrv] incubator-carbondata git commit: move package in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f75ddfb6/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --cc integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala
index 5ebb124,14a0930..ea52697
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala
@@@ -36,10 -37,10 +37,11 @@@ import org.apache.carbondata.common.Car
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.common.logging.impl.StandardLogService
import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.load.{BlockDetails, LoadMetadataDetails}
+import org.apache.carbondata.core.updatestatus.LoadMetadataDetails
import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory}
import org.apache.carbondata.processing.constants.DataProcessorConstants
- import org.apache.carbondata.processing.csvreaderstep.{BlockDetails, JavaRddIterator, RddInputUtils}
++import org.apache.carbondata.processing.csvreaderstep.{BlockDetails, RddInputUtils}
+ import org.apache.carbondata.processing.csvreaderstep.RddInputUtils
import org.apache.carbondata.processing.etl.DataLoadingException
import org.apache.carbondata.processing.graphgenerator.GraphGenerator
import org.apache.carbondata.processing.model.CarbonLoadModel
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f75ddfb6/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f75ddfb6/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --cc integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
index 645294f,46e83a5..77dc78a
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/NewCarbonDataLoadRDD.scala
@@@ -45,7 -43,6 +45,7 @@@ import org.apache.carbondata.core.updat
import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory}
import org.apache.carbondata.hadoop.csv.CSVInputFormat
import org.apache.carbondata.hadoop.csv.recorditerator.RecordReaderIterator
- import org.apache.carbondata.processing.csvreaderstep.{BlockDetails, JavaRddIterator}
++import org.apache.carbondata.processing.csvreaderstep.BlockDetails
import org.apache.carbondata.processing.model.CarbonLoadModel
import org.apache.carbondata.processing.newflow.DataLoadExecutor
import org.apache.carbondata.processing.newflow.exception.BadRecordFoundException
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f75ddfb6/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f75ddfb6/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f75ddfb6/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/CsvInput.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f75ddfb6/processing/src/main/java/org/apache/carbondata/processing/model/CarbonLoadModel.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f75ddfb6/processing/src/main/java/org/apache/carbondata/processing/newflow/CarbonDataLoadConfiguration.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f75ddfb6/processing/src/main/java/org/apache/carbondata/processing/newflow/DataLoadProcessBuilder.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f75ddfb6/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/f75ddfb6/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java
----------------------------------------------------------------------
[29/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalMaxMinShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalMaxMinShort.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalMaxMinShort.java
deleted file mode 100644
index cdf205c..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalMaxMinShort.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.nondecimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNonDecimalMaxMinShort extends ValueCompressionHolder<short[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNonDecimalMaxMinShort.class.getName());
- /**
- * compressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
- /**
- * value.
- */
- private short[] value;
-
- private MeasureDataChunkStore<short[]> measureChunkStore;
-
- private BigDecimal maxValue;
-
- private double divisionFactor;
-
- @Override public void setValue(short[] value) {
- this.value = value;
- }
-
- @Override public short[] getValue() { return this.value; }
-
- @Override
- public void uncompress(DataType dataType, byte[] compressedData,
- int offset, int length, int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, compressedData, offset, length);
- setUncompressedValues(value, decimalPlaces, maxValueObject);
- }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_SHORT, value);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToShortArray(buffer, value.length);
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long value is not defined for CompressionNonDecimalMaxMinShort");
- }
-
- @Override public double getDoubleValue(int index) {
- short shortValue = measureChunkStore.getShort(index);
- BigDecimal diff = BigDecimal.valueOf(shortValue / this.divisionFactor);
- return maxValue.subtract(diff).doubleValue();
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException("Get big decimal value is not supported");
- }
-
- private void setUncompressedValues(short[] data, int decimalPlaces, Object maxValueObject) {
- this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_SHORT, data.length);
- this.measureChunkStore.putData(data);
- this.maxValue = BigDecimal.valueOf((double) maxValueObject);
- this.divisionFactor = Math.pow(10, decimalPlaces);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalShort.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalShort.java
deleted file mode 100644
index 441a914..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/nondecimal/CompressionNonDecimalShort.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.nondecimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNonDecimalShort extends ValueCompressionHolder<short[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNonDecimalShort.class.getName());
- /**
- * shortCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
- /**
- * value.
- */
- private short[] value;
-
- private MeasureDataChunkStore<short[]> measureChunkStore;
-
- private double divisionFactory;
-
- @Override public void setValue(short[] value) {
- this.value = value;
- }
-
- @Override public short[] getValue() { return this.value; }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_SHORT, value);
- }
-
- @Override
- public void uncompress(DataType dataType, byte[] compressedData,
- int offset, int length, int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor,dataType,compressedData, offset, length);
- setUncompressedValues(value, decimalPlaces);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToShortArray(buffer, value.length);
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long value is not defined for CompressionNonDecimalShort");
- }
-
- @Override public double getDoubleValue(int index) {
- return (measureChunkStore.getShort(index) / this.divisionFactory);
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionNonDecimalShort");
- }
-
- private void setUncompressedValues(short[] data, int decimalPlaces) {
- this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_SHORT, data.length);
- this.measureChunkStore.putData(data);
- this.divisionFactory = Math.pow(10, decimalPlaces);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/CompressionNoneByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/CompressionNoneByte.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/CompressionNoneByte.java
deleted file mode 100644
index 5583a3c..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/CompressionNoneByte.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.none;
-
-import java.math.BigDecimal;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNoneByte extends ValueCompressionHolder<byte[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNoneByte.class.getName());
-
- /**
- * byteCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- /**
- * value.
- */
- private byte[] value;
-
- /**
- * actual data type
- */
- private DataType actualDataType;
-
- private MeasureDataChunkStore<byte[]> measureChunkStore;
-
- public CompressionNoneByte(DataType actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- @Override public void setValue(byte[] value) {
- this.value = value;
- }
-
- @Override
- public void uncompress(DataType dataType, byte[] data, int offset, int length,
- int mantissa, Object maxValueObject) {
- super.unCompress(compressor, dataType, data, offset, length);
- setUncompressedValues(value);
- }
-
- @Override public byte[] getValue() { return this.value; }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_BYTE, value);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- this.value = value;
- }
-
- @Override public long getLongValue(int index) {
- return measureChunkStore.getByte(index);
- }
-
- @Override public double getDoubleValue(int index) {
- return measureChunkStore.getByte(index);
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal is not defined for CompressionNoneByte");
- }
-
- private void setUncompressedValues(byte[] data) {
- this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_BYTE, data.length);
- this.measureChunkStore.putData(data);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/CompressionNoneDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/CompressionNoneDefault.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/CompressionNoneDefault.java
deleted file mode 100644
index b910872..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/CompressionNoneDefault.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.none;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNoneDefault extends ValueCompressionHolder<double[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNoneDefault.class.getName());
- /**
- * doubleCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
- /**
- * value.
- */
- private double[] value;
-
- private DataType actualDataType;
-
- private MeasureDataChunkStore<double[]> measureChunkStore;
-
- public CompressionNoneDefault(DataType actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- @Override public void setValue(double[] value) {this.value = value; }
-
- @Override
- public void uncompress(DataType dataType, byte[] data, int offset, int length,
- int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, data, offset, length);
- setUncompressedValues(value);
- }
-
- @Override public double[] getValue() { return this.value; }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_DOUBLE, value);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToDoubleArray(buffer, value.length);
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long value is not defined for CompressionNonDefault");
- }
-
- @Override public double getDoubleValue(int index) {
- return measureChunkStore.getDouble(index);
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal is not defined for CompressionNoneDefault");
- }
-
- private void setUncompressedValues(double[] data) {
- this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_DOUBLE, data.length);
- this.measureChunkStore.putData(data);
-
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/CompressionNoneInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/CompressionNoneInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/CompressionNoneInt.java
deleted file mode 100644
index ff88d44..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/CompressionNoneInt.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.none;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNoneInt extends ValueCompressionHolder<int[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNoneInt.class.getName());
- /**
- * intCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
- /**
- * value.
- */
- private int[] value;
-
- private DataType actualDataType;
-
- private MeasureDataChunkStore<int[]> measureChunkStore;
-
- public CompressionNoneInt(DataType actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- @Override public void setValue(int[] value) {
- this.value = value;
- }
-
- @Override public int[] getValue() { return this.value; }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_INT, value);
- }
-
- @Override
- public void uncompress(DataType dataType, byte[] data, int offset, int length,
- int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, data, offset, length);
- setUncompressedValues(value);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToIntArray(buffer, value.length);
- }
-
- @Override public long getLongValue(int index) {
- return measureChunkStore.getInt(index);
- }
-
- @Override public double getDoubleValue(int index) {
- return measureChunkStore.getInt(index);
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal is not defined for CompressionNoneLong");
- }
-
- private void setUncompressedValues(int[] data) {
- this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, data.length);
- this.measureChunkStore.putData(data);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/CompressionNoneLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/CompressionNoneLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/CompressionNoneLong.java
deleted file mode 100644
index ebd2ee4..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/CompressionNoneLong.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.none;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNoneLong extends ValueCompressionHolder<long[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNoneLong.class.getName());
- /**
- * longCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
- /**
- * value.
- */
- protected long[] value;
-
- private DataType actualDataType;
-
- private MeasureDataChunkStore<long[]> measureChunkStore;
-
- public CompressionNoneLong(DataType actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- @Override public void setValue(long[] value) { this.value = value; }
-
- @Override public long[] getValue() { return this.value; }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_LONG, value);
- }
-
- @Override
- public void uncompress(DataType dataType, byte[] data, int offset, int length,
- int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, data, offset, length);
- setUncompressedValues(value);
- }
-
- @Override public void setValueInBytes(byte[] byteValue) {
- ByteBuffer buffer = ByteBuffer.wrap(byteValue);
- this.value = ValueCompressionUtil.convertToLongArray(buffer, byteValue.length);
- }
-
- @Override public long getLongValue(int index) {
- return measureChunkStore.getLong(index);
- }
-
- @Override public double getDoubleValue(int index) {
- return measureChunkStore.getLong(index);
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException("Get big decimal is not supported");
- }
-
- private void setUncompressedValues(long[] data) {
- this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_LONG, data.length);
- this.measureChunkStore.putData(data);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/CompressionNoneShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/CompressionNoneShort.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/CompressionNoneShort.java
deleted file mode 100644
index 277fb24..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/none/CompressionNoneShort.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.compression.none;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastorage.store.compression.Compressor;
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNoneShort extends ValueCompressionHolder<short[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNoneShort.class.getName());
-
- /**
- * shortCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- /**
- * value.
- */
- private short[] shortValue;
-
- private MeasureDataChunkStore<short[]> measureChunkStore;
-
- private DataType actualDataType;
-
- public CompressionNoneShort(DataType actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- @Override public void setValue(short[] shortValue) {
- this.shortValue = shortValue;
- }
-
- @Override public short[] getValue() { return this.shortValue; }
-
- @Override
- public void uncompress(DataType dataType, byte[] data, int offset, int length,
- int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, data, offset, length);
- setUncompressedValues(shortValue);
- }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_SHORT, shortValue);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- shortValue = ValueCompressionUtil.convertToShortArray(buffer, value.length);
- }
-
- @Override public long getLongValue(int index) {
- return measureChunkStore.getShort(index);
- }
-
- @Override public double getDoubleValue(int index) {
- return measureChunkStore.getShort(index);
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal is not defined for CompressionNonShort");
- }
-
- private void setUncompressedValues(short[] data) {
- this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_SHORT, data.length);
- this.measureChunkStore.putData(data);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/CompressionBigDecimal.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/CompressionBigDecimal.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/CompressionBigDecimal.java
deleted file mode 100644
index db766ac..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/type/CompressionBigDecimal.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.datastorage.store.compression.type;
-
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.util.BigDecimalCompressionFinder;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-/**
- * Big decimal compression
- */
-public class CompressionBigDecimal<T> extends ValueCompressionHolder<T> {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionBigDecimal.class.getName());
-
- private BigDecimalCompressionFinder compressionFinder;
-
- /**
- * leftPart before decimal
- */
- private ValueCompressionHolder leftPart;
-
- /**
- * rightPart after decimal
- */
- private ValueCompressionHolder rightPart;
-
- private double divisionFactor;
-
- private boolean isDecimalPlacesNotZero;
-
- public CompressionBigDecimal(BigDecimalCompressionFinder compressionFinder,
- ValueCompressionHolder leftPart, ValueCompressionHolder rightPart) {
- this.compressionFinder = compressionFinder;
- this.leftPart = leftPart;
- this.rightPart = rightPart;
- }
-
- @Override public void setValue(T value) {
- Object[] values = (Object[]) value;
- leftPart.setValue(values[0]);
- rightPart.setValue(values[1]);
- }
-
- @Override
- public void uncompress(DataType dataType, byte[] data, int offset, int length,
- int decimalPlaces, Object maxValueObject) {
- if (decimalPlaces > 0) {
- this.isDecimalPlacesNotZero = true;
- }
- this.divisionFactor = Math.pow(10, decimalPlaces);
-
- ByteBuffer buffer = ByteBuffer.wrap(data, offset, length);
- int leftPathLength = buffer.getInt();
- int rightPartLength = length - leftPathLength - CarbonCommonConstants.INT_SIZE_IN_BYTE;
- Long[] maxValue = (Long[]) maxValueObject;
- leftPart.uncompress(compressionFinder.getLeftConvertedDataType(), data,
- offset + CarbonCommonConstants.INT_SIZE_IN_BYTE, leftPathLength, decimalPlaces,
- maxValue[0]);
- rightPart.uncompress(compressionFinder.getRightConvertedDataType(), data,
- offset + CarbonCommonConstants.INT_SIZE_IN_BYTE + leftPathLength, rightPartLength,
- decimalPlaces, maxValue[1]);
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long is not defined for CompressionBigDecimal");
- }
-
- @Override public double getDoubleValue(int index) {
- throw new UnsupportedOperationException(
- "Double is not defined for CompressionBigDecimal");
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- long leftValue = leftPart.getLongValue(index);
- long rightValue = 0;
- if (isDecimalPlacesNotZero) {
- rightValue = rightPart.getLongValue(index);
- }
- String decimalPart = Double.toString(rightValue / this.divisionFactor);
- String bigdStr = Long.toString(leftValue) + CarbonCommonConstants.POINT + decimalPart
- .substring(decimalPart.indexOf(".") + 1, decimalPart.length());
- return new BigDecimal(bigdStr);
- }
-
- @Override public T getValue() {
- Object[] values = new Object[2];
- values[0] = leftPart;
- values[1] = rightPart;
- return (T) values;
- }
-
- @Override public void setValueInBytes(byte[] value) {
- LOGGER.error("setValueInBytes() is not defined for CompressionBigDecimal");
- }
-
- @Override public void compress() {
- leftPart.compress();
- rightPart.compress();
- }
-
- @Override
- public byte[] getCompressedData() {
- byte[] leftdata = leftPart.getCompressedData();
- byte[] rightdata = rightPart.getCompressedData();
- ByteBuffer byteBuffer = ByteBuffer
- .allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE + leftdata.length
- + rightdata.length);
- byteBuffer.putInt(leftdata.length);
- byteBuffer.put(leftdata);
- byteBuffer.put(rightdata);
- byteBuffer.flip();
- return byteBuffer.array();
- }
-
- @Override public void freeMemory() {
- leftPart.freeMemory();
- rightPart.freeMemory();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/dataholder/CarbonReadDataHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/dataholder/CarbonReadDataHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/dataholder/CarbonReadDataHolder.java
deleted file mode 100644
index ff3df15..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/dataholder/CarbonReadDataHolder.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.dataholder;
-
-import java.math.BigDecimal;
-
-import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionHolder;
-
-// This class is used with Uncompressor to hold the decompressed column chunk in memory
-public class CarbonReadDataHolder {
-
- private ValueCompressionHolder unCompressValue;
-
- public CarbonReadDataHolder(ValueCompressionHolder unCompressValue) {
- this.unCompressValue = unCompressValue;
- }
-
- public long getReadableLongValueByIndex(int index) {
- return this.unCompressValue.getLongValue(index);
- }
-
- public BigDecimal getReadableBigDecimalValueByIndex(int index) {
- return this.unCompressValue.getBigDecimalValue(index);
- }
-
- public double getReadableDoubleValueByIndex(int index) {
- return this.unCompressValue.getDoubleValue(index);
- }
-
- public void freeMemory() {
- unCompressValue.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/dataholder/CarbonWriteDataHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/dataholder/CarbonWriteDataHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/dataholder/CarbonWriteDataHolder.java
deleted file mode 100644
index c87b266..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/dataholder/CarbonWriteDataHolder.java
+++ /dev/null
@@ -1,280 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.dataholder;
-
-public class CarbonWriteDataHolder {
- /**
- * doubleValues
- */
- private double[] doubleValues;
-
- /**
- * longValues
- */
- private long[] longValues;
-
- /**
- * bigDecimal left part
- */
- private long[] bigDecimalLeftValues;
-
- /**
- * bigDecimal right part
- */
- private long[] bigDecimalRightValues;
- /**
- * byteValues
- */
- private byte[][] byteValues;
-
- /**
- * byteValues for no dictionary and non kettle flow.
- */
- private byte[][][] byteValuesForNonDictionary;
-
- /**
- * byteValues
- */
- private byte[][][] columnByteValues;
-
- /**
- * size
- */
- private int size;
-
- /**
- * totalSize
- */
- private int totalSize;
-
- /**
- * Method to initialise double array
- *
- * @param size
- */
- public void initialiseDoubleValues(int size) {
- if (size < 1) {
- throw new IllegalArgumentException("Invalid array size");
- }
- doubleValues = new double[size];
- }
-
- public void reset() {
- size = 0;
- totalSize = 0;
- }
-
- /**
- * Method to initialise double array
- * TODO Remove after kettle flow got removed.
- *
- * @param size
- */
- public void initialiseByteArrayValues(int size) {
- if (size < 1) {
- throw new IllegalArgumentException("Invalid array size");
- }
-
- byteValues = new byte[size][];
- columnByteValues = new byte[size][][];
- }
-
- /**
- * Method to initialise byte array
- *
- * @param size
- */
- public void initialiseByteArrayValuesWithOutKettle(int size) {
- if (size < 1) {
- throw new IllegalArgumentException("Invalid array size");
- }
-
- byteValues = new byte[size][];
- }
-
- public void initialiseByteArrayValuesForNonDictionary(int size) {
- if (size < 1) {
- throw new IllegalArgumentException("Invalid array size");
- }
-
- byteValuesForNonDictionary = new byte[size][][];
- }
-
- /**
- * Method to initialise long array
- *
- * @param size
- */
- public void initialiseLongValues(int size) {
- if (size < 1) {
- throw new IllegalArgumentException("Invalid array size");
- }
- longValues = new long[size];
- }
-
- public void initialiseBigDecimalValues(int size) {
- if (size < 1) {
- throw new IllegalArgumentException("Invalid array size");
- }
- bigDecimalLeftValues = new long[size];
- bigDecimalRightValues = new long[size];
- }
-
- /**
- * set double value by index
- *
- * @param index
- * @param value
- */
- public void setWritableDoubleValueByIndex(int index, Object value) {
- doubleValues[index] = (Double) value;
- size++;
- }
-
- /**
- * set double value by index
- *
- * @param index
- * @param value
- */
- public void setWritableLongValueByIndex(int index, Object value) {
- longValues[index] = (Long) value;
- size++;
- }
-
- /**
- * set bigdecimal value by index
- *
- * @param index
- * @param value
- */
- public void setWritableBigDecimalValueByIndex(int index, long[] value) {
- bigDecimalLeftValues[index] = value[0];
- bigDecimalRightValues[index] = value[1];
- size++;
- }
- /**
- * set byte array value by index
- *
- * @param index
- * @param value
- */
- public void setWritableByteArrayValueByIndex(int index, byte[] value) {
- byteValues[index] = value;
- size++;
- if (null != value) totalSize += value.length;
- }
-
- public void setWritableNonDictByteArrayValueByIndex(int index, byte[][] value) {
- byteValuesForNonDictionary[index] = value;
- size++;
- if (null != value) totalSize += value.length;
- }
-
- /**
- * set byte array value by index
- */
- public void setWritableByteArrayValueByIndex(int index, int mdKeyIndex, Object[] columnData) {
- int l = 0;
- columnByteValues[index] = new byte[columnData.length - (mdKeyIndex + 1)][];
- for (int i = mdKeyIndex + 1; i < columnData.length; i++) {
- columnByteValues[index][l++] = (byte[]) columnData[i];
- }
- }
-
- /**
- * Get Writable Double Values
- */
- public double[] getWritableDoubleValues() {
- if (size < doubleValues.length) {
- double[] temp = new double[size];
- System.arraycopy(doubleValues, 0, temp, 0, size);
- doubleValues = temp;
- }
- return doubleValues;
- }
-
- /**
- * Get writable byte array values
- */
- public byte[] getWritableByteArrayValues() {
- byte[] temp = new byte[totalSize];
- int startIndexToCopy = 0;
- for (int i = 0; i < size; i++) {
- System.arraycopy(byteValues[i], 0, temp, startIndexToCopy, byteValues[i].length);
- startIndexToCopy += byteValues[i].length;
- }
- return temp;
- }
-
- public byte[][] getByteArrayValues() {
- if (size < byteValues.length) {
- byte[][] temp = new byte[size][];
- System.arraycopy(byteValues, 0, temp, 0, size);
- byteValues = temp;
- }
- return byteValues;
- }
-
- public byte[][][] getNonDictByteArrayValues() {
- if (size < byteValuesForNonDictionary.length) {
- byte[][][] temp = new byte[size][][];
- System.arraycopy(byteValuesForNonDictionary, 0, temp, 0, size);
- byteValuesForNonDictionary = temp;
- }
- return byteValuesForNonDictionary;
- }
-
- /**
- * Get Writable Double Values
- *
- * @return
- */
- public long[] getWritableLongValues() {
- if (size < longValues.length) {
- long[] temp = new long[size];
- System.arraycopy(longValues, 0, temp, 0, size);
- longValues = temp;
- }
- return longValues;
- }
-
- /**
- * Get Writable bigdecimal Values
- *
- * @return
- */
- public long[][] getWritableBigDecimalValues() {
- long[][] bigDecimalValues = new long[2][];
- if (size < bigDecimalLeftValues.length) {
- long[] temp = new long[size];
- System.arraycopy(bigDecimalLeftValues, 0, temp, 0, size);
- bigDecimalLeftValues = temp;
- }
- if (size < bigDecimalRightValues.length) {
- long[] temp = new long[size];
- System.arraycopy(bigDecimalRightValues, 0, temp, 0, size);
- bigDecimalRightValues = temp;
- }
- bigDecimalValues[0]= bigDecimalLeftValues;
- bigDecimalValues[1] = bigDecimalRightValues;
- return bigDecimalValues;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/AbstractDFSCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/AbstractDFSCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/AbstractDFSCarbonFile.java
deleted file mode 100644
index 26af405..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/AbstractDFSCarbonFile.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.filesystem;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
-import org.apache.carbondata.core.util.CarbonUtil;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-public abstract class AbstractDFSCarbonFile implements CarbonFile {
- /**
- * LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(AbstractDFSCarbonFile.class.getName());
- protected FileStatus fileStatus;
- protected FileSystem fs;
-
- public AbstractDFSCarbonFile(String filePath) {
- filePath = filePath.replace("\\", "/");
- Path path = new Path(filePath);
- try {
- fs = path.getFileSystem(FileFactory.getConfiguration());
- fileStatus = fs.getFileStatus(path);
- } catch (IOException e) {
- LOGGER.error("Exception occured:" + e.getMessage());
- }
- }
-
- public AbstractDFSCarbonFile(Path path) {
- try {
- fs = path.getFileSystem(FileFactory.getConfiguration());
- fileStatus = fs.getFileStatus(path);
- } catch (IOException e) {
- LOGGER.error("Exception occured:" + e.getMessage());
- }
- }
-
- public AbstractDFSCarbonFile(FileStatus fileStatus) {
- this.fileStatus = fileStatus;
- }
-
- @Override public boolean createNewFile() {
- Path path = fileStatus.getPath();
- try {
- return fs.createNewFile(path);
- } catch (IOException e) {
- return false;
- }
- }
-
- @Override public String getAbsolutePath() {
- return fileStatus.getPath().toString();
- }
-
- @Override public String getName() {
- return fileStatus.getPath().getName();
- }
-
- @Override public boolean isDirectory() {
- return fileStatus.isDirectory();
- }
-
- @Override public boolean exists() {
- try {
- if (null != fileStatus) {
- fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
- return fs.exists(fileStatus.getPath());
- }
- } catch (IOException e) {
- LOGGER.error("Exception occured:" + e.getMessage());
- }
- return false;
- }
-
- @Override public String getCanonicalPath() {
- return getAbsolutePath();
- }
-
- @Override public String getPath() {
- return getAbsolutePath();
- }
-
- @Override public long getSize() {
- return fileStatus.getLen();
- }
-
- public boolean renameTo(String changetoName) {
- FileSystem fs;
- try {
- fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
- return fs.rename(fileStatus.getPath(), new Path(changetoName));
- } catch (IOException e) {
- LOGGER.error("Exception occured:" + e.getMessage());
- return false;
- }
- }
-
- public boolean delete() {
- FileSystem fs;
- try {
- fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
- return fs.delete(fileStatus.getPath(), true);
- } catch (IOException e) {
- LOGGER.error("Exception occured:" + e.getMessage());
- return false;
- }
- }
-
- @Override public long getLastModifiedTime() {
- return fileStatus.getModificationTime();
- }
-
- @Override public boolean setLastModifiedTime(long timestamp) {
- try {
- fs.setTimes(fileStatus.getPath(), timestamp, timestamp);
- } catch (IOException e) {
- return false;
- }
- return true;
- }
-
- /**
- * This method will delete the data in file data from a given offset
- */
- @Override public boolean truncate(String fileName, long validDataEndOffset) {
- DataOutputStream dataOutputStream = null;
- DataInputStream dataInputStream = null;
- boolean fileTruncatedSuccessfully = false;
- // if bytes to read less than 1024 then buffer size should be equal to the given offset
- int bufferSize = validDataEndOffset > CarbonCommonConstants.BYTE_TO_KB_CONVERSION_FACTOR ?
- CarbonCommonConstants.BYTE_TO_KB_CONVERSION_FACTOR :
- (int) validDataEndOffset;
- // temporary file name
- String tempWriteFilePath = fileName + CarbonCommonConstants.TEMPWRITEFILEEXTENSION;
- FileFactory.FileType fileType = FileFactory.getFileType(fileName);
- try {
- CarbonFile tempFile = null;
- // delete temporary file if it already exists at a given path
- if (FileFactory.isFileExist(tempWriteFilePath, fileType)) {
- tempFile = FileFactory.getCarbonFile(tempWriteFilePath, fileType);
- tempFile.delete();
- }
- // create new temporary file
- FileFactory.createNewFile(tempWriteFilePath, fileType);
- tempFile = FileFactory.getCarbonFile(tempWriteFilePath, fileType);
- byte[] buff = new byte[bufferSize];
- dataInputStream = FileFactory.getDataInputStream(fileName, fileType);
- // read the data
- int read = dataInputStream.read(buff, 0, buff.length);
- dataOutputStream = FileFactory.getDataOutputStream(tempWriteFilePath, fileType);
- dataOutputStream.write(buff, 0, read);
- long remaining = validDataEndOffset - read;
- // anytime we should not cross the offset to be read
- while (remaining > 0) {
- if (remaining > bufferSize) {
- buff = new byte[bufferSize];
- } else {
- buff = new byte[(int) remaining];
- }
- read = dataInputStream.read(buff, 0, buff.length);
- dataOutputStream.write(buff, 0, read);
- remaining = remaining - read;
- }
- CarbonUtil.closeStreams(dataInputStream, dataOutputStream);
- // rename the temp file to original file
- tempFile.renameForce(fileName);
- fileTruncatedSuccessfully = true;
- } catch (IOException e) {
- LOGGER.error("Exception occured while truncating the file " + e.getMessage());
- } finally {
- CarbonUtil.closeStreams(dataOutputStream, dataInputStream);
- }
- return fileTruncatedSuccessfully;
- }
-
- /**
- * This method will be used to check whether a file has been modified or not
- *
- * @param fileTimeStamp time to be compared with latest timestamp of file
- * @param endOffset file length to be compared with current length of file
- * @return
- */
- @Override public boolean isFileModified(long fileTimeStamp, long endOffset) {
- boolean isFileModified = false;
- if (getLastModifiedTime() > fileTimeStamp || getSize() > endOffset) {
- isFileModified = true;
- }
- return isFileModified;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/AlluxioCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/AlluxioCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/AlluxioCarbonFile.java
deleted file mode 100644
index 9ff0f59..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/AlluxioCarbonFile.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.filesystem;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-
-
-
-public class AlluxioCarbonFile extends AbstractDFSCarbonFile {
- /**
- * LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(AlluxioCarbonFile.class.getName());
-
- public AlluxioCarbonFile(String filePath) {
- super(filePath);
- }
-
- public AlluxioCarbonFile(Path path) {
- super(path);
- }
-
- public AlluxioCarbonFile(FileStatus fileStatus) {
- super(fileStatus);
- }
-
- /**
- * @param listStatus
- * @return
- */
- private CarbonFile[] getFiles(FileStatus[] listStatus) {
- if (listStatus == null) {
- return new CarbonFile[0];
- }
- CarbonFile[] files = new CarbonFile[listStatus.length];
- for (int i = 0; i < files.length; i++) {
- files[i] = new AlluxioCarbonFile(listStatus[i]);
- }
- return files;
- }
-
- @Override
- public CarbonFile[] listFiles() {
- FileStatus[] listStatus = null;
- try {
- if (null != fileStatus && fileStatus.isDirectory()) {
- Path path = fileStatus.getPath();
- listStatus = path.getFileSystem(FileFactory.getConfiguration()).listStatus(path);
- } else {
- return null;
- }
- } catch (IOException e) {
- LOGGER.error("Exception occured: " + e.getMessage());
- return new CarbonFile[0];
- }
- return getFiles(listStatus);
- }
-
- @Override
- public CarbonFile[] listFiles(final CarbonFileFilter fileFilter) {
- CarbonFile[] files = listFiles();
- if (files != null && files.length >= 1) {
- List<CarbonFile> fileList = new ArrayList<CarbonFile>(files.length);
- for (int i = 0; i < files.length; i++) {
- if (fileFilter.accept(files[i])) {
- fileList.add(files[i]);
- }
- }
- if (fileList.size() >= 1) {
- return fileList.toArray(new CarbonFile[fileList.size()]);
- } else {
- return new CarbonFile[0];
- }
- }
- return files;
- }
-
- @Override
- public CarbonFile getParentFile() {
- Path parent = fileStatus.getPath().getParent();
- return null == parent ? null : new AlluxioCarbonFile(parent);
- }
-
- @Override
- public boolean renameForce(String changetoName) {
- FileSystem fs;
- try {
- fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
- if (fs instanceof DistributedFileSystem) {
- ((DistributedFileSystem) fs).rename(fileStatus.getPath(), new Path(changetoName),
- org.apache.hadoop.fs.Options.Rename.OVERWRITE);
- return true;
- } else {
- return false;
- }
- } catch (IOException e) {
- LOGGER.error("Exception occured: " + e.getMessage());
- return false;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/CarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/CarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/CarbonFile.java
deleted file mode 100644
index 642055b..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/CarbonFile.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.filesystem;
-
-public interface CarbonFile {
-
- String getAbsolutePath();
-
- CarbonFile[] listFiles(CarbonFileFilter fileFilter);
-
- CarbonFile[] listFiles();
-
- String getName();
-
- boolean isDirectory();
-
- boolean exists();
-
- String getCanonicalPath();
-
- CarbonFile getParentFile();
-
- String getPath();
-
- long getSize();
-
- boolean renameTo(String changetoName);
-
- boolean renameForce(String changetoName);
-
- boolean delete();
-
- boolean createNewFile();
-
- long getLastModifiedTime();
-
- boolean setLastModifiedTime(long timestamp);
-
- boolean truncate(String fileName, long validDataEndOffset);
-
- /**
- * This method will be used to check whether a file has been modified or not
- *
- * @param fileTimeStamp time to be compared with latest timestamp of file
- * @param endOffset file length to be compared with current length of file
- * @return
- */
- boolean isFileModified(long fileTimeStamp, long endOffset);
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/CarbonFileFilter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/CarbonFileFilter.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/CarbonFileFilter.java
deleted file mode 100644
index 7db3b2b..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/CarbonFileFilter.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.filesystem;
-
-public interface CarbonFileFilter {
- boolean accept(CarbonFile file);
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/HDFSCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/HDFSCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/HDFSCarbonFile.java
deleted file mode 100644
index ebe18e4..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/HDFSCarbonFile.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.filesystem;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
-
-public class HDFSCarbonFile extends AbstractDFSCarbonFile {
- /**
- * LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(HDFSCarbonFile.class.getName());
-
- public HDFSCarbonFile(String filePath) {
- super(filePath);
- }
-
- public HDFSCarbonFile(Path path) {
- super(path);
- }
-
- public HDFSCarbonFile(FileStatus fileStatus) {
- super(fileStatus);
- }
-
- /**
- * @param listStatus
- * @return
- */
- private CarbonFile[] getFiles(FileStatus[] listStatus) {
- if (listStatus == null) {
- return new CarbonFile[0];
- }
- CarbonFile[] files = new CarbonFile[listStatus.length];
- for (int i = 0; i < files.length; i++) {
- files[i] = new HDFSCarbonFile(listStatus[i]);
- }
- return files;
- }
-
- @Override
- public CarbonFile[] listFiles() {
- FileStatus[] listStatus = null;
- try {
- if (null != fileStatus && fileStatus.isDirectory()) {
- Path path = fileStatus.getPath();
- listStatus = path.getFileSystem(FileFactory.getConfiguration()).listStatus(path);
- } else {
- return null;
- }
- } catch (IOException e) {
- LOGGER.error("Exception occured: " + e.getMessage());
- return new CarbonFile[0];
- }
- return getFiles(listStatus);
- }
-
- @Override
- public CarbonFile[] listFiles(final CarbonFileFilter fileFilter) {
- CarbonFile[] files = listFiles();
- if (files != null && files.length >= 1) {
- List<CarbonFile> fileList = new ArrayList<CarbonFile>(files.length);
- for (int i = 0; i < files.length; i++) {
- if (fileFilter.accept(files[i])) {
- fileList.add(files[i]);
- }
- }
- if (fileList.size() >= 1) {
- return fileList.toArray(new CarbonFile[fileList.size()]);
- } else {
- return new CarbonFile[0];
- }
- }
- return files;
- }
-
- @Override
- public CarbonFile getParentFile() {
- Path parent = fileStatus.getPath().getParent();
- return null == parent ? null : new HDFSCarbonFile(parent);
- }
-
- @Override
- public boolean renameForce(String changetoName) {
- FileSystem fs;
- try {
- fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
- if (fs instanceof DistributedFileSystem) {
- ((DistributedFileSystem) fs).rename(fileStatus.getPath(), new Path(changetoName),
- org.apache.hadoop.fs.Options.Rename.OVERWRITE);
- return true;
- } else {
- return false;
- }
- } catch (IOException e) {
- LOGGER.error("Exception occured: " + e.getMessage());
- return false;
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/LocalCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/LocalCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/LocalCarbonFile.java
deleted file mode 100644
index 406f6d1..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/LocalCarbonFile.java
+++ /dev/null
@@ -1,229 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.filesystem;
-
-import java.io.File;
-import java.io.FileFilter;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.channels.FileChannel;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
-import org.apache.carbondata.core.util.CarbonUtil;
-
-import org.apache.hadoop.fs.Path;
-
-public class LocalCarbonFile implements CarbonFile {
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(LocalCarbonFile.class.getName());
- private File file;
-
- public LocalCarbonFile(String filePath) {
- Path pathWithoutSchemeAndAuthority = Path.getPathWithoutSchemeAndAuthority(new Path(filePath));
- file = new File(pathWithoutSchemeAndAuthority.toString());
- }
-
- public LocalCarbonFile(File file) {
- this.file = file;
- }
-
- @Override public String getAbsolutePath() {
- return file.getAbsolutePath();
- }
-
- @Override public CarbonFile[] listFiles(final CarbonFileFilter fileFilter) {
- if (!file.isDirectory()) {
- return null;
- }
-
- File[] files = file.listFiles(new FileFilter() {
-
- @Override public boolean accept(File pathname) {
- return fileFilter.accept(new LocalCarbonFile(pathname));
- }
- });
-
- if (files == null) {
- return new CarbonFile[0];
- }
-
- CarbonFile[] carbonFiles = new CarbonFile[files.length];
-
- for (int i = 0; i < carbonFiles.length; i++) {
- carbonFiles[i] = new LocalCarbonFile(files[i]);
- }
-
- return carbonFiles;
- }
-
- @Override public String getName() {
- return file.getName();
- }
-
- @Override public boolean isDirectory() {
- return file.isDirectory();
- }
-
- @Override public boolean exists() {
- if (file != null) {
- return file.exists();
- }
- return false;
- }
-
- @Override public String getCanonicalPath() {
- try {
- return file.getCanonicalPath();
- } catch (IOException e) {
- LOGGER
- .error(e, "Exception occured" + e.getMessage());
- }
- return null;
- }
-
- @Override public CarbonFile getParentFile() {
- return new LocalCarbonFile(file.getParentFile());
- }
-
- @Override public String getPath() {
- return file.getPath();
- }
-
- @Override public long getSize() {
- return file.length();
- }
-
- public boolean renameTo(String changetoName) {
- return file.renameTo(new File(changetoName));
- }
-
- public boolean delete() {
- return file.delete();
- }
-
- @Override public CarbonFile[] listFiles() {
-
- if (!file.isDirectory()) {
- return null;
- }
- File[] files = file.listFiles();
- if (files == null) {
- return new CarbonFile[0];
- }
- CarbonFile[] carbonFiles = new CarbonFile[files.length];
- for (int i = 0; i < carbonFiles.length; i++) {
- carbonFiles[i] = new LocalCarbonFile(files[i]);
- }
-
- return carbonFiles;
-
- }
-
- @Override public boolean createNewFile() {
- try {
- return file.createNewFile();
- } catch (IOException e) {
- return false;
- }
- }
-
- @Override public long getLastModifiedTime() {
- return file.lastModified();
- }
-
- @Override public boolean setLastModifiedTime(long timestamp) {
- return file.setLastModified(timestamp);
- }
-
- /**
- * This method will delete the data in file data from a given offset
- */
- @Override public boolean truncate(String fileName, long validDataEndOffset) {
- FileChannel source = null;
- FileChannel destination = null;
- boolean fileTruncatedSuccessfully = false;
- // temporary file name
- String tempWriteFilePath = fileName + CarbonCommonConstants.TEMPWRITEFILEEXTENSION;
- FileFactory.FileType fileType = FileFactory.getFileType(fileName);
- try {
- CarbonFile tempFile = null;
- // delete temporary file if it already exists at a given path
- if (FileFactory.isFileExist(tempWriteFilePath, fileType)) {
- tempFile = FileFactory.getCarbonFile(tempWriteFilePath, fileType);
- tempFile.delete();
- }
- // create new temporary file
- FileFactory.createNewFile(tempWriteFilePath, fileType);
- tempFile = FileFactory.getCarbonFile(tempWriteFilePath, fileType);
- source = new FileInputStream(fileName).getChannel();
- destination = new FileOutputStream(tempWriteFilePath).getChannel();
- long read = destination.transferFrom(source, 0, validDataEndOffset);
- long totalBytesRead = read;
- long remaining = validDataEndOffset - totalBytesRead;
- // read till required data offset is not reached
- while (remaining > 0) {
- read = destination.transferFrom(source, totalBytesRead, remaining);
- totalBytesRead = totalBytesRead + read;
- remaining = remaining - totalBytesRead;
- }
- CarbonUtil.closeStreams(source, destination);
- // rename the temp file to original file
- tempFile.renameForce(fileName);
- fileTruncatedSuccessfully = true;
- } catch (IOException e) {
- LOGGER.error("Exception occured while truncating the file " + e.getMessage());
- } finally {
- CarbonUtil.closeStreams(source, destination);
- }
- return fileTruncatedSuccessfully;
- }
-
- /**
- * This method will be used to check whether a file has been modified or not
- *
- * @param fileTimeStamp time to be compared with latest timestamp of file
- * @param endOffset file length to be compared with current length of file
- * @return
- */
- @Override public boolean isFileModified(long fileTimeStamp, long endOffset) {
- boolean isFileModified = false;
- if (getLastModifiedTime() > fileTimeStamp || getSize() > endOffset) {
- isFileModified = true;
- }
- return isFileModified;
- }
-
- @Override public boolean renameForce(String changetoName) {
- File destFile = new File(changetoName);
- if (destFile.exists()) {
- if (destFile.delete()) {
- return file.renameTo(new File(changetoName));
- }
- }
-
- return file.renameTo(new File(changetoName));
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/ViewFSCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/ViewFSCarbonFile.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/ViewFSCarbonFile.java
deleted file mode 100644
index 8f11b7a..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/filesystem/ViewFSCarbonFile.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.datastorage.store.filesystem;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
-
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.viewfs.ViewFileSystem;
-
-public class ViewFSCarbonFile extends AbstractDFSCarbonFile {
- /**
- * LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(ViewFSCarbonFile.class.getName());
-
- public ViewFSCarbonFile(String filePath) {
- super(filePath);
- }
-
- public ViewFSCarbonFile(Path path) {
- super(path);
- }
-
- public ViewFSCarbonFile(FileStatus fileStatus) {
- super(fileStatus);
- }
-
- /**
- * @param listStatus
- * @return
- */
- private CarbonFile[] getFiles(FileStatus[] listStatus) {
- if (listStatus == null) {
- return new CarbonFile[0];
- }
- CarbonFile[] files = new CarbonFile[listStatus.length];
- for (int i = 0; i < files.length; i++) {
- files[i] = new ViewFSCarbonFile(listStatus[i]);
- }
- return files;
- }
-
- @Override
- public CarbonFile[] listFiles() {
- FileStatus[] listStatus = null;
- try {
- if (null != fileStatus && fileStatus.isDirectory()) {
- Path path = fileStatus.getPath();
- listStatus = path.getFileSystem(FileFactory.getConfiguration()).listStatus(path);
- } else {
- return null;
- }
- } catch (IOException ex) {
- LOGGER.error("Exception occured" + ex.getMessage());
- return new CarbonFile[0];
- }
- return getFiles(listStatus);
- }
-
- @Override
- public CarbonFile[] listFiles(final CarbonFileFilter fileFilter) {
- CarbonFile[] files = listFiles();
- if (files != null && files.length >= 1) {
- List<CarbonFile> fileList = new ArrayList<CarbonFile>(files.length);
- for (int i = 0; i < files.length; i++) {
- if (fileFilter.accept(files[i])) {
- fileList.add(files[i]);
- }
- }
- if (fileList.size() >= 1) {
- return fileList.toArray(new CarbonFile[fileList.size()]);
- } else {
- return new CarbonFile[0];
- }
- }
- return files;
- }
-
- @Override public CarbonFile getParentFile() {
- Path parent = fileStatus.getPath().getParent();
- return null == parent ? null : new ViewFSCarbonFile(parent);
- }
-
- @Override
- public boolean renameForce(String changetoName) {
- FileSystem fs;
- try {
- fs = fileStatus.getPath().getFileSystem(FileFactory.getConfiguration());
- if (fs instanceof ViewFileSystem) {
- fs.delete(new Path(changetoName), true);
- fs.rename(fileStatus.getPath(), new Path(changetoName));
- return true;
- } else {
- return false;
- }
- } catch (IOException e) {
- LOGGER.error("Exception occured" + e.getMessage());
- return false;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/CompressedDataMeasureDataWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/CompressedDataMeasureDataWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/CompressedDataMeasureDataWrapper.java
deleted file mode 100644
index c9571b2..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/CompressedDataMeasureDataWrapper.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.store.impl;
-
-import org.apache.carbondata.core.datastorage.store.MeasureDataWrapper;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
-
-public class CompressedDataMeasureDataWrapper implements MeasureDataWrapper {
-
- private final CarbonReadDataHolder[] values;
-
- public CompressedDataMeasureDataWrapper(final CarbonReadDataHolder[] values) {
- this.values = values;
- }
-
- @Override public CarbonReadDataHolder[] getValues() {
- return values;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/DFSFileHolderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/DFSFileHolderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/DFSFileHolderImpl.java
deleted file mode 100644
index 6e2f5d7..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/impl/DFSFileHolderImpl.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.core.datastorage.store.impl;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-
-public class DFSFileHolderImpl implements FileHolder {
- /**
- * cache to hold filename and its stream
- */
- private Map<String, FSDataInputStream> fileNameAndStreamCache;
-
- public DFSFileHolderImpl() {
- this.fileNameAndStreamCache =
- new HashMap<String, FSDataInputStream>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
- }
-
- @Override public byte[] readByteArray(String filePath, long offset, int length)
- throws IOException {
- FSDataInputStream fileChannel = updateCache(filePath);
- return read(fileChannel, length, offset);
- }
-
- /**
- * This method will be used to check whether stream is already present in
- * cache or not for filepath if not present then create it and then add to
- * cache, other wise get from cache
- *
- * @param filePath fully qualified file path
- * @return channel
- */
- private FSDataInputStream updateCache(String filePath) throws IOException {
- FSDataInputStream fileChannel = fileNameAndStreamCache.get(filePath);
- if (null == fileChannel) {
- Path pt = new Path(filePath);
- FileSystem fs = FileSystem.get(FileFactory.getConfiguration());
- fileChannel = fs.open(pt);
- fileNameAndStreamCache.put(filePath, fileChannel);
- }
- return fileChannel;
- }
-
- /**
- * This method will be used to read from file based on number of bytes to be read and positon
- *
- * @param channel file channel
- * @param size number of bytes
- * @param offset position
- * @return byte buffer
- */
- private byte[] read(FSDataInputStream channel, int size, long offset) throws IOException {
- byte[] byteBffer = new byte[size];
- channel.seek(offset);
- channel.readFully(byteBffer);
- return byteBffer;
- }
-
- /**
- * This method will be used to read from file based on number of bytes to be read and positon
- *
- * @param channel file channel
- * @param size number of bytes
- * @return byte buffer
- */
- private byte[] read(FSDataInputStream channel, int size) throws IOException {
- byte[] byteBffer = new byte[size];
- channel.readFully(byteBffer);
- return byteBffer;
- }
-
- @Override public int readInt(String filePath, long offset) throws IOException {
- FSDataInputStream fileChannel = updateCache(filePath);
- fileChannel.seek(offset);
- return fileChannel.readInt();
- }
-
- @Override public long readDouble(String filePath, long offset) throws IOException {
- FSDataInputStream fileChannel = updateCache(filePath);
- fileChannel.seek(offset);
- return fileChannel.readLong();
- }
-
- @Override public void finish() throws IOException {
- for (Entry<String, FSDataInputStream> entry : fileNameAndStreamCache.entrySet()) {
- FSDataInputStream channel = entry.getValue();
- if (null != channel) {
- channel.close();
- }
- }
- }
-
- @Override public byte[] readByteArray(String filePath, int length) throws IOException {
- FSDataInputStream fileChannel = updateCache(filePath);
- return read(fileChannel, length);
- }
-
- @Override public long readLong(String filePath, long offset) throws IOException {
- FSDataInputStream fileChannel = updateCache(filePath);
- fileChannel.seek(offset);
- return fileChannel.readLong();
- }
-
- @Override public int readInt(String filePath) throws IOException {
- FSDataInputStream fileChannel = updateCache(filePath);
- return fileChannel.readInt();
- }
-}
[52/57] [abbrv] incubator-carbondata git commit: fix style
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/WriterCompressModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/WriterCompressModel.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/WriterCompressModel.java
deleted file mode 100644
index d833c61..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/WriterCompressModel.java
+++ /dev/null
@@ -1,220 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.compression;
-
-import org.apache.carbondata.core.util.CompressionFinder;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-
-public class WriterCompressModel {
-
- /**
- * DataType[] variable.
- */
- private ValueCompressionUtil.DataType[] convertedDataType;
- /**
- * DataType[] variable.
- */
- private ValueCompressionUtil.DataType[] actualDataType;
-
- /**
- * maxValue
- */
- private Object[] maxValue;
- /**
- * minValue.
- */
- private Object[] minValue;
-
- /**
- * uniqueValue
- */
- private Object[] uniqueValue;
- /**
- * mantissa.
- */
- private int[] mantissa;
-
- /**
- * aggType
- */
- private char[] type;
-
- /**
- * dataTypeSelected
- */
- private byte[] dataTypeSelected;
- /**
- * unCompressValues.
- */
- private ValueCompressionHolder[] valueHolder;
-
- private CompressionFinder[] compressionFinders;
-
- /**
- * @return the convertedDataType
- */
- public ValueCompressionUtil.DataType[] getConvertedDataType() {
- return convertedDataType;
- }
-
- /**
- * @param convertedDataType the convertedDataType to set
- */
- public void setConvertedDataType(ValueCompressionUtil.DataType[] convertedDataType) {
- this.convertedDataType = convertedDataType;
- }
-
- /**
- * @return the actualDataType
- */
- public ValueCompressionUtil.DataType[] getActualDataType() {
- return actualDataType;
- }
-
- /**
- * @param actualDataType
- */
- public void setActualDataType(ValueCompressionUtil.DataType[] actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- /**
- * @return the maxValue
- */
- public Object[] getMaxValue() {
- return maxValue;
- }
-
- /**
- * @param maxValue the maxValue to set
- */
- public void setMaxValue(Object[] maxValue) {
- this.maxValue = maxValue;
- }
-
- /**
- * @return the mantissa
- */
- public int[] getMantissa() {
- return mantissa;
- }
-
- /**
- * @param mantissa the mantissa to set
- */
- public void setMantissa(int[] mantissa) {
- this.mantissa = mantissa;
- }
-
- /**
- * getUnCompressValues().
- *
- * @return the unCompressValues
- */
- public ValueCompressionHolder[] getValueCompressionHolder() {
- return valueHolder;
- }
-
- /**
- * @param valueHolder set the ValueCompressionHolder
- */
- public void setValueCompressionHolder(ValueCompressionHolder[] valueHolder) {
- this.valueHolder = valueHolder;
- }
-
- /**
- * getMinValue
- *
- * @return
- */
- public Object[] getMinValue() {
- return minValue;
- }
-
- /**
- * setMinValue.
- *
- * @param minValue
- */
- public void setMinValue(Object[] minValue) {
- this.minValue = minValue;
- }
-
- /**
- * @return the aggType
- */
- public char[] getType() {
- return type;
- }
-
- /**
- * @param type the type to set
- */
- public void setType(char[] type) {
- this.type = type;
- }
-
- /**
- * @return the dataTypeSelected
- */
- public byte[] getDataTypeSelected() {
- return dataTypeSelected;
- }
-
- /**
- * @param dataTypeSelected the dataTypeSelected to set
- */
- public void setDataTypeSelected(byte[] dataTypeSelected) {
- this.dataTypeSelected = dataTypeSelected;
- }
-
- /**
- * getUniqueValue
- *
- * @return
- */
- public Object[] getUniqueValue() {
- return uniqueValue;
- }
-
- /**
- * setUniqueValue
- *
- * @param uniqueValue
- */
- public void setUniqueValue(Object[] uniqueValue) {
- this.uniqueValue = uniqueValue;
- }
-
- public void setCompressionFinders(CompressionFinder[] compressionFinders) {
- this.compressionFinders = compressionFinders;
- }
-
- public CompressionFinder[] getCompressionFinders() {
- return this.compressionFinders;
- }
-
- /**
- * @return the compType
- */
- public ValueCompressionUtil.COMPRESSION_TYPE getCompType(int index) {
- return this.compressionFinders[index].getCompType();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinByte.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinByte.java
deleted file mode 100644
index bd7ff59..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinByte.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
-3 * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.compression.decimal;
-
-import java.math.BigDecimal;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionMaxMinByte extends ValueCompressionHolder<byte[]> {
-
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionMaxMinByte.class.getName());
-
- /**
- * compressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- /**
- * value.
- */
- protected byte[] value;
-
- private MeasureDataChunkStore<byte[]> measureChunkStore;
-
- /**
- * actual data type
- */
- protected DataType actualDataType;
-
- private double maxValue;
-
- public CompressionMaxMinByte(DataType actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- @Override public byte[] getValue() {return this.value; }
-
- @Override public void setValue(byte[] value) {
- this.value = value;
- }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_BYTE, value);
- }
-
- @Override
- public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
- int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, compressedData, offset, length);
- setUncompressedValues(value, maxValueObject);
-
- }
-
- @Override public void setValueInBytes(byte[] value) {
- this.value = value;
- }
-
- @Override public long getLongValue(int index) {
- byte byteValue = measureChunkStore.getByte(index);
- return (long) (maxValue - byteValue);
- }
-
- @Override public double getDoubleValue(int index) {
- byte byteValue = measureChunkStore.getByte(index);
- return (maxValue - byteValue);
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionMaxMinByte");
- }
-
- private void setUncompressedValues(byte[] data, Object maxValueObject) {
- this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_BYTE, data.length);
- this.measureChunkStore.putData(data);
- if (maxValueObject instanceof Long) {
- this.maxValue = (long)maxValueObject;
- } else {
- this.maxValue = (double) maxValueObject;
- }
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinDefault.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinDefault.java
deleted file mode 100644
index 145f2ac..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinDefault.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.compression.decimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-
-public class CompressionMaxMinDefault extends ValueCompressionHolder<double[]> {
-
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionMaxMinDefault.class.getName());
-
- /**
- * compressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- /**
- * value.
- */
- private double[] value;
-
- private MeasureDataChunkStore<double[]> measureChunkStore;
-
- /**
- * actual data type
- */
- private DataType actualDataType;
-
- private double maxValue;
-
- public CompressionMaxMinDefault(DataType actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- @Override public void setValue(double[] value) {
- this.value = value;
- }
-
- @Override public double[] getValue() {return this.value; }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_DOUBLE, value);
- }
-
- @Override public void uncompress(DataType dataType, byte[] compressedData,
- int offset, int length, int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, compressedData, offset, length);
- setUncompressedValues(value, maxValueObject);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToDoubleArray(buffer, value.length);
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long value is not defined for CompressionMaxMinDefault");
- }
-
- @Override public double getDoubleValue(int index) {
- double doubleValue = measureChunkStore.getDouble(index);
- return maxValue - doubleValue;
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionMaxMinDefault");
- }
-
- private void setUncompressedValues(double[] data, Object maxValueObject) {
- this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_DOUBLE, data.length);
- this.measureChunkStore.putData(data);
- if (maxValueObject instanceof Long) {
- this.maxValue = (long) maxValueObject;
- } else {
- this.maxValue = (double) maxValueObject;
- }
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinInt.java
deleted file mode 100644
index fcafb0f..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinInt.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.compression.decimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionMaxMinInt extends ValueCompressionHolder<int[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionMaxMinInt.class.getName());
-
- /**
- * compressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- private MeasureDataChunkStore<int[]> measureChunkStore;
-
- /**
- * value.
- */
- private int[] value;
-
- private DataType actualDataType;
-
- private double maxValue;
-
- public CompressionMaxMinInt(DataType actualType) {
- this.actualDataType = actualType;
- }
-
- @Override public void setValue(int[] value) {
- this.value = value;
- }
-
- @Override public int[] getValue() { return this.value; }
-
- @Override
- public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
- int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, compressedData, offset, length);
- setUncompressedValues(value, maxValueObject);
- }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_INT, value);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToIntArray(buffer, value.length);
- }
-
- @Override public long getLongValue(int index) {
- int intValue = measureChunkStore.getInt(index);
- return (long) (maxValue - intValue);
- }
-
- @Override public double getDoubleValue(int index) {
- int intValue = measureChunkStore.getInt(index);
- return maxValue - intValue;
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionMaxMinInt");
- }
-
- private void setUncompressedValues(int[] data, Object maxValueObject) {
- this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, data.length);
- this.measureChunkStore.putData(data);
- if (maxValueObject instanceof Long) {
- this.maxValue = (long) maxValueObject;
- } else {
- this.maxValue = (double) maxValueObject;
- }
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinLong.java
deleted file mode 100644
index 3be00e4..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinLong.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.compression.decimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionMaxMinLong extends ValueCompressionHolder<long[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionMaxMinLong.class.getName());
- /**
- * compressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- private MeasureDataChunkStore<long[]> measureChunkStore;
-
- /**
- * value.
- */
- protected long[] value;
-
- protected DataType actualDataType;
-
- private double maxValue;
-
- public CompressionMaxMinLong(DataType actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_LONG, value);
- }
-
- @Override public void setValue(long[] value) {
- this.value = value;
-
- }
-
- @Override
- public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
- int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, compressedData, offset, length);
- setUncompressValues(value, maxValueObject);
- }
-
- @Override public long[] getValue() {return this.value; }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToLongArray(buffer, value.length);
- }
-
- @Override public long getLongValue(int index) {
- long longValue = measureChunkStore.getLong(index);
- return (long) maxValue - longValue;
- }
-
- @Override public double getDoubleValue(int index) {
- long longValue = measureChunkStore.getLong(index);
- return maxValue - longValue;
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionMaxMinLong");
- }
-
- private void setUncompressValues(long[] data, Object maxValueObject) {
- this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_LONG, data.length);
- this.measureChunkStore.putData(data);
- if (maxValueObject instanceof Long) {
- this.maxValue = (long) maxValueObject;
- } else {
- this.maxValue = (double) maxValueObject;
- }
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinShort.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinShort.java
deleted file mode 100644
index 546f903..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinShort.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.compression.decimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionMaxMinShort extends ValueCompressionHolder<short[]> {
-
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionMaxMinShort.class.getName());
-
- /**
- * shortCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- private MeasureDataChunkStore<short[]> measureChunkStore;
-
- /**
- * value.
- */
- private short[] value;
-
- private DataType actualDataType;
-
- private double maxValue;
-
- public CompressionMaxMinShort(DataType actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- @Override public void setValue(short[] value) {
- this.value = value;
-
- }
-
- @Override
- public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
- int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, compressedData, offset, length);
- setUncompressedValues(value, maxValueObject);
- }
-
- @Override public short[] getValue() {return this.value; }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_SHORT, value);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToShortArray(buffer, value.length);
- }
-
- @Override public long getLongValue(int index) {
- short shortValue = measureChunkStore.getShort(index);
- return (long) maxValue - shortValue;
- }
-
- @Override public double getDoubleValue(int index) {
- short shortValue = measureChunkStore.getShort(index);
- return maxValue - shortValue;
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionMaxMinShort");
- }
-
- private void setUncompressedValues(short[] data, Object maxValueObject) {
- this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_SHORT, data.length);
- this.measureChunkStore.putData(data);
- if (maxValueObject instanceof Long) {
- this.maxValue = (long) maxValueObject;
- } else {
- this.maxValue = (double) maxValueObject;
- }
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalByte.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalByte.java
deleted file mode 100644
index 7aec7fc..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalByte.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.compression.nondecimal;
-
-import java.math.BigDecimal;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNonDecimalByte extends ValueCompressionHolder<byte[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNonDecimalByte.class.getName());
-
- /**
- * compressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- /**
- * value.
- */
- private byte[] value;
-
- private MeasureDataChunkStore<byte[]> measureChunkStore;
-
- private double divisionFactory;
-
- @Override public void setValue(byte[] value) {
- this.value = value;
- }
-
- @Override public byte[] getValue() {return this.value; }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_BYTE, value);
- }
-
- @Override
- public void uncompress(DataType dataType, byte[] compressedData,
- int offset, int length, int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, compressedData, offset, length);
- setUncompressedValues(value, decimalPlaces);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- this.value = value;
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long value is not defined for CompressionNonDecimalByte");
- }
-
- @Override public double getDoubleValue(int index) {
- return (measureChunkStore.getByte(index) / this.divisionFactory);
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionNonDecimalByte");
- }
-
- private void setUncompressedValues(byte[] data, int decimalPlaces) {
- this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_BYTE, data.length);
- this.measureChunkStore.putData(data);
- this.divisionFactory = Math.pow(10, decimalPlaces);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalDefault.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalDefault.java
deleted file mode 100644
index 7a3ec18..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalDefault.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.compression.nondecimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNonDecimalDefault extends ValueCompressionHolder<double[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNonDecimalDefault.class.getName());
- /**
- * doubleCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
- /**
- * value.
- */
- private double[] value;
-
- private MeasureDataChunkStore<double[]> measureChunkStore;
-
- private double divisionFactory;
-
- @Override
- public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
- int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, compressedData, offset, length);
- setUncompressedValues(value, decimalPlaces);
- }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_DOUBLE, value);
- }
-
- @Override public void setValue(double[] value) {
- this.value = value;
-
- }
-
- @Override public double[] getValue() {return this.value; }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToDoubleArray(buffer, value.length);
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long value is not defined for CompressionNonDecimalDefault");
- }
-
- @Override public double getDoubleValue(int index) {
- return (measureChunkStore.getDouble(index) / divisionFactory);
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionNonDecimalDefault");
- }
-
- private void setUncompressedValues(double[] data, int decimalPlaces) {
- this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_DOUBLE, data.length);
- this.measureChunkStore.putData(data);
- this.divisionFactory = Math.pow(10, decimalPlaces);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalInt.java
deleted file mode 100644
index 2968702..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalInt.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.compression.nondecimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNonDecimalInt extends ValueCompressionHolder<int[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNonDecimalInt.class.getName());
- /**
- * intCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
- /**
- * value.
- */
- private int[] value;
-
- private MeasureDataChunkStore<int[]> measureChunkStore;
-
- private double divisionFactory;
-
- @Override public void setValue(int[] value) {
- this.value = value;
- }
-
- @Override public int[] getValue() { return this.value; }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_INT, value);
- }
-
- @Override public void setValueInBytes(byte[] bytesArr) {
- ByteBuffer buffer = ByteBuffer.wrap(bytesArr);
- this.value = ValueCompressionUtil.convertToIntArray(buffer, bytesArr.length);
- }
-
- @Override
- public void uncompress(DataType dataType, byte[] compressedData, int offset, int length,
- int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, compressedData, offset, length);
- setUncompressedValues(value, decimalPlaces);
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long value is not defined for CompressionNonDecimalInt");
- }
-
- @Override public double getDoubleValue(int index) {
- return (measureChunkStore.getInt(index) / this.divisionFactory);
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionNonDecmialInt");
- }
-
- private void setUncompressedValues(int[] data, int decimalPlaces) {
- this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, data.length);
- this.measureChunkStore.putData(data);
- this.divisionFactory = Math.pow(10, decimalPlaces);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalLong.java
deleted file mode 100644
index 0083736..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalLong.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.compression.nondecimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNonDecimalLong extends ValueCompressionHolder<long[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNonDecimalLong.class.getName());
-
- /**
- * longCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- /**
- * value.
- */
- private long[] value;
-
- private MeasureDataChunkStore<long[]> measureChunkStore;
-
- private double divisionFactory;
-
- @Override public void setValue(long[] value) {
- this.value = value;
- }
-
- @Override public long[] getValue() { return this.value; }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_LONG, value);
- }
-
- @Override public void uncompress(DataType dataType, byte[] compressedData, int offset,
- int length, int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, compressedData, offset, length);
- setUncompressedValues(value, decimalPlaces);
- }
-
- @Override public void setValueInBytes(byte[] bytes) {
- ByteBuffer buffer = ByteBuffer.wrap(bytes);
- this.value = ValueCompressionUtil.convertToLongArray(buffer, bytes.length);
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long value is not defined for CompressionNonDecimalLong");
- }
-
- @Override public double getDoubleValue(int index) {
- return (measureChunkStore.getLong(index) / this.divisionFactory);
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionNonDecimalLong");
- }
-
- private void setUncompressedValues(long[] data, int decimalPlaces) {
- this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_LONG, data.length);
- this.measureChunkStore.putData(data);
- this.divisionFactory = Math.pow(10, decimalPlaces);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinByte.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinByte.java
deleted file mode 100644
index 74d67d1..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinByte.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.compression.nondecimal;
-
-import java.math.BigDecimal;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNonDecimalMaxMinByte extends ValueCompressionHolder<byte[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNonDecimalMaxMinByte.class.getName());
-
- /**
- * compressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- /**
- * value.
- */
- private byte[] value;
-
- private MeasureDataChunkStore<byte[]> measureChunkStore;
-
- private BigDecimal maxValue;
-
- private double divisionFactor;
-
- @Override
- public void uncompress(DataType dataType, byte[] compressedData,
- int offset, int length, int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, compressedData, offset, length);
- setUncompressedValues(value, decimalPlaces, maxValueObject );
- }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_BYTE, value);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- this.value = value;
- }
-
- @Override public void setValue(byte[] value) {
- this.value = value;
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long value is not defined for CompressionNonDecimalMaxMinByte");
- }
-
- @Override public double getDoubleValue(int index) {
- byte byteValue = measureChunkStore.getByte(index);
- BigDecimal diff = BigDecimal.valueOf(byteValue / this.divisionFactor);
- return maxValue.subtract(diff).doubleValue();
-
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionNonDecimalMaxMinByte");
- }
-
- private void setUncompressedValues(byte[] data, int decimalPlaces, Object maxValueObject) {
- this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_BYTE, data.length);
- this.measureChunkStore.putData(data);
- this.maxValue = BigDecimal.valueOf((double) maxValueObject);
- this.divisionFactor = Math.pow(10, decimalPlaces);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-
- @Override public byte[] getValue() { return this.value; }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java
deleted file mode 100644
index e8c386f..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.compression.nondecimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNonDecimalMaxMinDefault extends ValueCompressionHolder<double[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNonDecimalMaxMinDefault.class.getName());
-
- /**
- * doubleCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- /**
- * value.
- */
- private double[] value;
-
- private MeasureDataChunkStore<double[]> measureChunkStore;
-
- private BigDecimal maxValue;
-
- private double divisionFactor;
-
- @Override public void setValue(double[] value) {
- this.value = value;
- }
-
- @Override public double[] getValue() { return this.value; }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_DOUBLE, value);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToDoubleArray(buffer, value.length);
- }
-
- @Override
- public void uncompress(DataType dataType, byte[] compressedData,
- int offset, int length, int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor,dataType,compressedData, offset, length);
- setUncompressedValues(value, decimalPlaces, maxValueObject);
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long value is not defined for CompressionNonDecimalMaxMinDefault");
- }
-
- @Override public double getDoubleValue(int index) {
- double doubleValue = measureChunkStore.getDouble(index);
- BigDecimal diff = BigDecimal.valueOf(doubleValue / this.divisionFactor);
- return maxValue.subtract(diff).doubleValue();
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionNonDecimalMaxMinDefault");
- }
-
- private void setUncompressedValues(double[] data, int decimalPlaces, Object maxValueObject) {
- this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_DOUBLE, data.length);
- this.measureChunkStore.putData(data);
- this.maxValue = BigDecimal.valueOf((double) maxValueObject);
- this.divisionFactor = Math.pow(10, decimalPlaces);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinInt.java
deleted file mode 100644
index fc0f815..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinInt.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.compression.nondecimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNonDecimalMaxMinInt extends ValueCompressionHolder<int[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNonDecimalMaxMinInt.class.getName());
- /**
- * intCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
- /**
- * value.
- */
- private int[] value;
-
- private MeasureDataChunkStore<int[]> measureChunkStore;
-
- private double divisionFactor;
-
- private BigDecimal maxValue;
-
- @Override public void setValue(int[] value) {
- this.value = value;
- }
-
- @Override public int[] getValue() {return this.value; }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_INT, value);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToIntArray(buffer, value.length);
- }
-
- @Override
- public void uncompress(DataType dataType, byte[] compressedData,
- int offset, int length, int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, compressedData, offset, length);
- setUncompressedValues(value, decimalPlaces, maxValueObject);
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long value is not defined for CompressionNonDecimalMaxMinInt");
- }
-
- @Override public double getDoubleValue(int index) {
- int intValue = measureChunkStore.getInt(index);
- BigDecimal diff = BigDecimal.valueOf(intValue / this.divisionFactor);
- return maxValue.subtract(diff).doubleValue();
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionNonDecimalMaxMinInt");
- }
-
- private void setUncompressedValues(int[] data, int decimalPlaces, Object maxValueObject) {
- this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, data.length);
- this.measureChunkStore.putData(data);
- this.maxValue = BigDecimal.valueOf((double) maxValueObject);
- this.divisionFactor = Math.pow(10, decimalPlaces);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinLong.java
deleted file mode 100644
index ddffc2b..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinLong.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.compression.nondecimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNonDecimalMaxMinLong extends ValueCompressionHolder<long[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNonDecimalMaxMinLong.class.getName());
-
- /**
- * longCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
- /**
- * value.
- */
- private long[] value;
-
- private MeasureDataChunkStore<long[]> measureChunkStore;
-
- private BigDecimal maxValue;
-
- private double divisionFactor;
-
- @Override public void setValue(long[] value) {
- this.value = value;
-
- }
-
- @Override public long[] getValue() { return this.value; }
-
- @Override
- public void uncompress(DataType dataType, byte[] compressedData,
- int offset, int length, int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, compressedData, offset, length);
- setUncompressedValues(value, decimalPlaces, maxValueObject);
- }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_LONG, value);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buff = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToLongArray(buff, value.length);
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long value is not defined for CompressionNonDecimalMaxMinLong");
- }
-
- @Override public double getDoubleValue(int index) {
- long longValue = measureChunkStore.getLong(index);
- BigDecimal diff = BigDecimal.valueOf(longValue / this.divisionFactor);
- return maxValue.subtract(diff).doubleValue();
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException("Big decimal value is not supported");
- }
-
- private void setUncompressedValues(long[] data, int decimalPlaces, Object maxValueObject) {
- this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_LONG, data.length);
- this.measureChunkStore.putData(data);
- this.maxValue = BigDecimal.valueOf((double) maxValueObject);
- this.divisionFactor = Math.pow(10, decimalPlaces);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinShort.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinShort.java
deleted file mode 100644
index 1306b1f..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinShort.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.compression.nondecimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNonDecimalMaxMinShort extends ValueCompressionHolder<short[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNonDecimalMaxMinShort.class.getName());
- /**
- * compressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
- /**
- * value.
- */
- private short[] value;
-
- private MeasureDataChunkStore<short[]> measureChunkStore;
-
- private BigDecimal maxValue;
-
- private double divisionFactor;
-
- @Override public void setValue(short[] value) {
- this.value = value;
- }
-
- @Override public short[] getValue() { return this.value; }
-
- @Override
- public void uncompress(DataType dataType, byte[] compressedData,
- int offset, int length, int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, compressedData, offset, length);
- setUncompressedValues(value, decimalPlaces, maxValueObject);
- }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_SHORT, value);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToShortArray(buffer, value.length);
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long value is not defined for CompressionNonDecimalMaxMinShort");
- }
-
- @Override public double getDoubleValue(int index) {
- short shortValue = measureChunkStore.getShort(index);
- BigDecimal diff = BigDecimal.valueOf(shortValue / this.divisionFactor);
- return maxValue.subtract(diff).doubleValue();
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException("Get big decimal value is not supported");
- }
-
- private void setUncompressedValues(short[] data, int decimalPlaces, Object maxValueObject) {
- this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_SHORT, data.length);
- this.measureChunkStore.putData(data);
- this.maxValue = BigDecimal.valueOf((double) maxValueObject);
- this.divisionFactor = Math.pow(10, decimalPlaces);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalShort.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalShort.java
deleted file mode 100644
index 334d7e8..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalShort.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.compression.nondecimal;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNonDecimalShort extends ValueCompressionHolder<short[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNonDecimalShort.class.getName());
- /**
- * shortCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
- /**
- * value.
- */
- private short[] value;
-
- private MeasureDataChunkStore<short[]> measureChunkStore;
-
- private double divisionFactory;
-
- @Override public void setValue(short[] value) {
- this.value = value;
- }
-
- @Override public short[] getValue() { return this.value; }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_SHORT, value);
- }
-
- @Override
- public void uncompress(DataType dataType, byte[] compressedData,
- int offset, int length, int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor,dataType,compressedData, offset, length);
- setUncompressedValues(value, decimalPlaces);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToShortArray(buffer, value.length);
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long value is not defined for CompressionNonDecimalShort");
- }
-
- @Override public double getDoubleValue(int index) {
- return (measureChunkStore.getShort(index) / this.divisionFactory);
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal value is not defined for CompressionNonDecimalShort");
- }
-
- private void setUncompressedValues(short[] data, int decimalPlaces) {
- this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_SHORT, data.length);
- this.measureChunkStore.putData(data);
- this.divisionFactory = Math.pow(10, decimalPlaces);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneByte.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneByte.java
deleted file mode 100644
index 32d3e63..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneByte.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.compression.none;
-
-import java.math.BigDecimal;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNoneByte extends ValueCompressionHolder<byte[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNoneByte.class.getName());
-
- /**
- * byteCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
-
- /**
- * value.
- */
- private byte[] value;
-
- /**
- * actual data type
- */
- private DataType actualDataType;
-
- private MeasureDataChunkStore<byte[]> measureChunkStore;
-
- public CompressionNoneByte(DataType actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- @Override public void setValue(byte[] value) {
- this.value = value;
- }
-
- @Override
- public void uncompress(DataType dataType, byte[] data, int offset, int length,
- int mantissa, Object maxValueObject) {
- super.unCompress(compressor, dataType, data, offset, length);
- setUncompressedValues(value);
- }
-
- @Override public byte[] getValue() { return this.value; }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_BYTE, value);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- this.value = value;
- }
-
- @Override public long getLongValue(int index) {
- return measureChunkStore.getByte(index);
- }
-
- @Override public double getDoubleValue(int index) {
- return measureChunkStore.getByte(index);
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal is not defined for CompressionNoneByte");
- }
-
- private void setUncompressedValues(byte[] data) {
- this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_BYTE, data.length);
- this.measureChunkStore.putData(data);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneDefault.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneDefault.java
deleted file mode 100644
index dad6bdb..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneDefault.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.compression.none;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNoneDefault extends ValueCompressionHolder<double[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNoneDefault.class.getName());
- /**
- * doubleCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
- /**
- * value.
- */
- private double[] value;
-
- private DataType actualDataType;
-
- private MeasureDataChunkStore<double[]> measureChunkStore;
-
- public CompressionNoneDefault(DataType actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- @Override public void setValue(double[] value) {this.value = value; }
-
- @Override
- public void uncompress(DataType dataType, byte[] data, int offset, int length,
- int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, data, offset, length);
- setUncompressedValues(value);
- }
-
- @Override public double[] getValue() { return this.value; }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_DOUBLE, value);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToDoubleArray(buffer, value.length);
- }
-
- @Override public long getLongValue(int index) {
- throw new UnsupportedOperationException(
- "Long value is not defined for CompressionNonDefault");
- }
-
- @Override public double getDoubleValue(int index) {
- return measureChunkStore.getDouble(index);
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal is not defined for CompressionNoneDefault");
- }
-
- private void setUncompressedValues(double[] data) {
- this.measureChunkStore = MeasureChunkStoreFactory.INSTANCE
- .getMeasureDataChunkStore(DataType.DATA_DOUBLE, data.length);
- this.measureChunkStore.putData(data);
-
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/48316190/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneInt.java
deleted file mode 100644
index 13ecd8c..0000000
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneInt.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.core.datastorage.compression.none;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.util.ValueCompressionUtil;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
-
-public class CompressionNoneInt extends ValueCompressionHolder<int[]> {
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CompressionNoneInt.class.getName());
- /**
- * intCompressor.
- */
- private static Compressor compressor = CompressorFactory.getInstance().getCompressor();
- /**
- * value.
- */
- private int[] value;
-
- private DataType actualDataType;
-
- private MeasureDataChunkStore<int[]> measureChunkStore;
-
- public CompressionNoneInt(DataType actualDataType) {
- this.actualDataType = actualDataType;
- }
-
- @Override public void setValue(int[] value) {
- this.value = value;
- }
-
- @Override public int[] getValue() { return this.value; }
-
- @Override public void compress() {
- compressedValue = super.compress(compressor, DataType.DATA_INT, value);
- }
-
- @Override
- public void uncompress(DataType dataType, byte[] data, int offset, int length,
- int decimalPlaces, Object maxValueObject) {
- super.unCompress(compressor, dataType, data, offset, length);
- setUncompressedValues(value);
- }
-
- @Override public void setValueInBytes(byte[] value) {
- ByteBuffer buffer = ByteBuffer.wrap(value);
- this.value = ValueCompressionUtil.convertToIntArray(buffer, value.length);
- }
-
- @Override public long getLongValue(int index) {
- return measureChunkStore.getInt(index);
- }
-
- @Override public double getDoubleValue(int index) {
- return measureChunkStore.getInt(index);
- }
-
- @Override public BigDecimal getBigDecimalValue(int index) {
- throw new UnsupportedOperationException(
- "Big decimal is not defined for CompressionNoneLong");
- }
-
- private void setUncompressedValues(int[] data) {
- this.measureChunkStore =
- MeasureChunkStoreFactory.INSTANCE.getMeasureDataChunkStore(DataType.DATA_INT, data.length);
- this.measureChunkStore.putData(data);
- }
-
- @Override public void freeMemory() {
- this.measureChunkStore.freeMemory();
- }
-}
[07/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java
new file mode 100644
index 0000000..bfc4f9a
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/GreaterThanExpressionUnitTest.java
@@ -0,0 +1,364 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.expression.conditional;
+
+import java.math.BigDecimal;
+import java.sql.Timestamp;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.RowImpl;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.apache.spark.sql.types.Decimal;
+import org.junit.Test;
+
+import static junit.framework.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class GreaterThanExpressionUnitTest {
+ static GreaterThanExpression greaterThanExpression;
+
+ @Test public void testEvaluateForGreaterThanExpressionWithStringDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
+ right.setColIndex(1);
+ greaterThanExpression = new GreaterThanExpression(left, right);
+ RowImpl value = new RowImpl();
+ String[] row = { "string1" };
+ String[] row1 = { "String's Value" };
+ Object objectRow[] = { row, row1 };
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public String getString() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return "string1";
+
+ } else {
+ return "String's Value";
+
+ }
+
+ }
+ };
+ value.setValues(objectRow);
+ ExpressionResult result = greaterThanExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForGreaterThanExpressionWithShortDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("id", DataType.SHORT);
+ left.setColIndex(1);
+ greaterThanExpression = new GreaterThanExpression(left, right);
+ RowImpl value = new RowImpl();
+ Short[] row = { 170 };
+ Short[] row1 = { 70 };
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Short getShort() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 170;
+
+ } else {
+ return 70;
+
+ }
+
+ }
+ };
+
+ ExpressionResult result = greaterThanExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+
+ }
+
+ @Test public void testEvaluateForGreaterThanExpressionWithDoubleDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("right_contact", DataType.DOUBLE);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("left_contact", DataType.DOUBLE);
+ left.setColIndex(1);
+ greaterThanExpression = new GreaterThanExpression(left, right);
+ RowImpl value = new RowImpl();
+ Double[] row = { 44D };
+ Double[] row1 = { 20D };
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Double getDouble() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 44D;
+
+ } else {
+ return 20D;
+
+ }
+
+ }
+ };
+
+ ExpressionResult result = greaterThanExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForGreaterThanExpressionWithIntDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("right_number", DataType.INT);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("left_number", DataType.INT);
+ left.setColIndex(1);
+ greaterThanExpression = new GreaterThanExpression(left, right);
+ RowImpl value = new RowImpl();
+ Integer[] row = { 140 };
+ Integer[] row1 = { 150 };
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Integer getInt() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 150;
+
+ } else {
+ return 140;
+
+ }
+
+ }
+ };
+
+ ExpressionResult result = greaterThanExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForGreaterThanExpressionWithTimestampDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ try {
+ ColumnExpression left = new ColumnExpression("timestamp", DataType.TIMESTAMP);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("timestamp", DataType.TIMESTAMP);
+ right.setColIndex(1);
+
+ greaterThanExpression = new GreaterThanExpression(left, right);
+
+ RowImpl value = new RowImpl();
+
+ DateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy");
+
+ Date date = dateFormat.parse("23/09/2007");
+ long time = date.getTime();
+ Timestamp[] row = { new Timestamp(time) };
+
+ Date date1 = dateFormat.parse("24/09/2007");
+ long time1 = date1.getTime();
+ Timestamp[] row1 = { new Timestamp(time1) };
+
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Long getTime() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 1190592000L;
+ } else {
+ return 1190505600L;
+ }
+ }
+ };
+
+ ExpressionResult result = greaterThanExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ } catch (ParseException e) {
+ System.out.println("Error while parsing " + e.getMessage());
+ }
+ }
+
+ @Test public void testEvaluateForGreaterThanExpressionWithLongDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("contact", DataType.LONG);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("contact", DataType.LONG);
+ left.setColIndex(1);
+ greaterThanExpression = new GreaterThanExpression(left, right);
+ RowImpl value = new RowImpl();
+ Long[] row = { 1234567654321L };
+ Long[] row1 = { 123456765432234L };
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Long getLong() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 123456765432234L;
+ } else {
+ return 1234567654321L;
+ }
+ }
+ };
+
+ ExpressionResult result = greaterThanExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForGreaterThanExpressionWithDecimalDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("contact", DataType.DECIMAL);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("contact", DataType.DECIMAL);
+ left.setColIndex(1);
+ greaterThanExpression = new GreaterThanExpression(left, right);
+ RowImpl value = new RowImpl();
+ Decimal[] row = new Decimal[] { Decimal.apply(12345.0) };
+ Decimal[] row1 = new Decimal[] { Decimal.apply(123451245.0) };
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public BigDecimal getDecimal() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return new BigDecimal(123451245.0);
+ } else {
+ return new BigDecimal(12345.0);
+ }
+ }
+ };
+
+ ExpressionResult result = greaterThanExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test(expected = FilterUnsupportedException.class) public void testForGreaterThanExpressionWithDefaultCase()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("contact", DataType.BOOLEAN);
+ right.setColIndex(0);
+ greaterThanExpression = new GreaterThanExpression(right, right);
+ RowImpl value = new RowImpl();
+ Boolean[] row = { true };
+ Object objectRow[] = { row };
+ value.setValues(objectRow);
+ greaterThanExpression.evaluate(value);
+ }
+
+ @Test public void testEvaluateForGreaterThanExpressionWithIsNullReturnTrue()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
+ right.setColIndex(0);
+ greaterThanExpression = new GreaterThanExpression(right, right);
+ RowImpl value = new RowImpl();
+ Short[] row = { 15 };
+ Object objectRow[] = { row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public boolean isNull() {
+ return true;
+ }
+ };
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Short getShort() {
+ return 15;
+ }
+ };
+
+ ExpressionResult result = greaterThanExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+
+ }
+
+ @Test public void testEvaluateForGreaterThanExpressionWithLeftAndRightDifferentDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("name", DataType.STRING);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("number", DataType.INT);
+ right.setColIndex(1);
+ greaterThanExpression = new GreaterThanExpression(left, right);
+ RowImpl value = new RowImpl();
+ String[] row = { "String1" };
+ Integer[] row1 = { 14 };
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Integer getInt() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 15;
+ } else {
+ return 14;
+
+ }
+
+ }
+ };
+
+ ExpressionResult result = greaterThanExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testForGreaterThanExpressionWithGetString() throws Exception {
+ ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
+ left.setColIndex(0);
+ greaterThanExpression = new GreaterThanExpression(left, right);
+ String expected_result = "GreaterThan(ColumnExpression(left_name),ColumnExpression(right_name))";
+ String result = greaterThanExpression.getString();
+ assertEquals(expected_result, result);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/InExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/InExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/InExpressionUnitTest.java
new file mode 100644
index 0000000..d7d5e78
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/InExpressionUnitTest.java
@@ -0,0 +1,276 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.expression.conditional;
+
+import java.math.BigDecimal;
+import java.sql.Timestamp;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.RowImpl;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.apache.spark.sql.types.Decimal;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class InExpressionUnitTest {
+
+ static InExpression inExpression;
+
+ @Test public void testEvaluateForInExpressionWithString()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
+ right.setColIndex(1);
+ inExpression = new InExpression(left, right);
+ RowImpl value = new RowImpl();
+ String row = "string1";
+ String row1 = "string1";
+ Object objectRow[] = { row, row1 };
+
+ new MockUp<ExpressionResult>() {
+
+ @Mock public DataType getDataType() {
+ return DataType.STRING;
+ }
+
+ @Mock public String getString() {
+ return "string1";
+ }
+ };
+
+ value.setValues(objectRow);
+ ExpressionResult result = inExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForInExpressionWithShortDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+
+ ColumnExpression left = new ColumnExpression("left_id", DataType.SHORT);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_id", DataType.SHORT);
+ right.setColIndex(1);
+ inExpression = new InExpression(left, right);
+ RowImpl value = new RowImpl();
+ Short row = 150;
+ Short row1 = 150;
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Short getShort() {
+ return 150;
+ }
+ };
+
+ ExpressionResult result = inExpression.evaluate(value);
+ assertEquals(result.getDataType(), DataType.BOOLEAN);
+
+ }
+
+ @Test public void testEvaluateForInExpressionWithIntDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+
+ ColumnExpression left = new ColumnExpression("left_id", DataType.INT);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_id", DataType.INT);
+ right.setColIndex(1);
+ inExpression = new InExpression(left, right);
+ RowImpl value = new RowImpl();
+ Integer row = 15052;
+ Integer row1 = 15052;
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Integer getInt() {
+ return 15052;
+ }
+ };
+
+ ExpressionResult result = inExpression.evaluate(value);
+ assertEquals(result.getDataType(), DataType.BOOLEAN);
+
+ }
+
+ @Test public void testEvaluateForInExpressionWithDoubleDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("left_contact", DataType.DOUBLE);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_contact", DataType.DOUBLE);
+ right.setColIndex(1);
+ inExpression = new InExpression(left, right);
+ RowImpl value = new RowImpl();
+ Double row = 44521D;
+ Double row1 = 44521D;
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Double getDouble() {
+ return 44521D;
+ }
+ };
+
+ ExpressionResult result = inExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForInExpressionWithLongDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("left_contact", DataType.LONG);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_contact", DataType.LONG);
+ right.setColIndex(1);
+ inExpression = new InExpression(left, right);
+ RowImpl value = new RowImpl();
+ Long row = 1234567654321L;
+ Long row1 = 1234567654321L;
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Long getLong() {
+ return 1234567654321L;
+ }
+ };
+
+ ExpressionResult result = inExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForInExpressionWithTimestampDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ try {
+ ColumnExpression left = new ColumnExpression("left_timestamp", DataType.TIMESTAMP);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_timestamp", DataType.TIMESTAMP);
+ right.setColIndex(1);
+ inExpression = new InExpression(left, right);
+
+ RowImpl value = new RowImpl();
+ DateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy");
+ Date date = dateFormat.parse("23/09/2007");
+
+ long time = date.getTime();
+
+ Timestamp row = new Timestamp(time);
+ Timestamp row1 = new Timestamp(time);
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Long getTime() {
+ return 18465213000000L;
+ }
+ };
+
+ ExpressionResult result = inExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+ } catch (ParseException e) {
+ System.out.println("Error while parsing " + e.getMessage());
+ }
+ }
+
+ @Test public void testEvaluateForInExpressionWithDecimalDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("left_contact", DataType.DECIMAL);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_contact", DataType.DECIMAL);
+ right.setColIndex(1);
+ inExpression = new InExpression(left, right);
+ RowImpl value = new RowImpl();
+ Decimal row = Decimal.apply(123452154.0);
+ Decimal row1 = Decimal.apply(123452154.0);
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public BigDecimal getDecimal() {
+ return new BigDecimal(123452154.0);
+ }
+ };
+
+ ExpressionResult result = inExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test(expected = FilterUnsupportedException.class) public void testForInExpressionWithDefaultCase()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("contact", DataType.BOOLEAN);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("contact", DataType.BOOLEAN);
+ right.setColIndex(1);
+ inExpression = new InExpression(left, right);
+ RowImpl value = new RowImpl();
+ Boolean row = true;
+ Boolean row1 = true;
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+ inExpression.evaluate(value);
+ }
+
+ @Test public void testForInExpressionWithGetString() throws Exception {
+ ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
+ right.setColIndex(1);
+ inExpression = new InExpression(left, right);
+ String expected_result = "IN(ColumnExpression(left_name),ColumnExpression(right_name))";
+ String result = inExpression.getString();
+ assertEquals(expected_result, result);
+ }
+
+ @Test public void testEvaluateForInExpressionWithLeftAndRightDifferentDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("name", DataType.STRING);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("number", DataType.INT);
+ left.setColIndex(1);
+ inExpression = new InExpression(left, right);
+ RowImpl value = new RowImpl();
+ String row1 = "String1";
+ Integer row = 14523 ;
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Integer getInt() {
+ return 14523;
+ }
+ };
+
+ ExpressionResult result = inExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java
new file mode 100644
index 0000000..d2d46a8
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanEqualToExpressionUnitTest.java
@@ -0,0 +1,368 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.expression.conditional;
+
+import java.math.BigDecimal;
+import java.sql.Timestamp;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.RowImpl;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.apache.spark.sql.types.Decimal;
+import org.junit.Test;
+
+import static junit.framework.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class LessThanEqualToExpressionUnitTest {
+
+ static LessThanEqualToExpression lessThanEqualToExpression;
+
+ @Test public void testEvaluateForLessThanEqualToExpressionWithBothStringISSame()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
+ left.setColIndex(1);
+ lessThanEqualToExpression = new LessThanEqualToExpression(left, left);
+ RowImpl value = new RowImpl();
+ String[] row = { "String is Value" };
+ String[] row1 = { "string1" };
+ Object objectRow[] = { row, row1 };
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public String getString() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return "String is Value";
+
+ } else {
+ return "string1";
+
+ }
+
+ }
+ };
+
+ value.setValues(objectRow);
+ ExpressionResult result = lessThanEqualToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForLessThanEqualToExpressionWithShortDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("id", DataType.SHORT);
+ left.setColIndex(1);
+ lessThanEqualToExpression = new LessThanEqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ Short[] row = { 1550 };
+ Short[] row1 = { 3365 };
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Short getShort() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 1550;
+
+ } else {
+ return 3365;
+
+ }
+
+ }
+ };
+
+ ExpressionResult result = lessThanEqualToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+
+ }
+
+ @Test public void testEvaluateForLessThanEqualToExpressionWithDoubleDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("right_contact", DataType.DOUBLE);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("left_contact", DataType.DOUBLE);
+ left.setColIndex(1);
+ lessThanEqualToExpression = new LessThanEqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ Double[] row = { 4852.2D };
+ Double[] row1 = { 4852.2D };
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Double getDouble() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 4852.2D;
+
+ } else {
+ return 4852.2D;
+
+ }
+
+ }
+ };
+
+ ExpressionResult result = lessThanEqualToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForLessThanEqualToExpressionWithIntDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("right_number", DataType.INT);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("left_number", DataType.INT);
+ left.setColIndex(1);
+ lessThanEqualToExpression = new LessThanEqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ Integer[] row = { 144580 };
+ Integer[] row1 = { 14500 };
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Integer getInt() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 14500;
+
+ } else {
+ return 144580;
+
+ }
+
+ }
+ };
+
+ ExpressionResult result = lessThanEqualToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForLessThanEqualToExpressionWithTimestampDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ try {
+ ColumnExpression left = new ColumnExpression("timestamp", DataType.TIMESTAMP);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("timestamp", DataType.TIMESTAMP);
+ right.setColIndex(1);
+
+ lessThanEqualToExpression = new LessThanEqualToExpression(left, right);
+
+ RowImpl value = new RowImpl();
+
+ DateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy");
+
+ Date date = dateFormat.parse("23/09/2007");
+ long time = date.getTime();
+ Timestamp[] row = { new Timestamp(time) };
+
+ Date date1 = dateFormat.parse("24/09/2007");
+ long time1 = date1.getTime();
+ Timestamp[] row1 = { new Timestamp(time1) };
+
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Long getTime() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 1190505600L;
+ } else {
+ return 1190592000L;
+ }
+ }
+ };
+
+ ExpressionResult result = lessThanEqualToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ } catch (ParseException e) {
+ System.out.println("Error while parsing " + e.getMessage());
+ }
+ }
+
+ @Test public void testEvaluateForLessThanEqualToExpressionWithLongDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("right_contact", DataType.LONG);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("left_contact", DataType.LONG);
+ left.setColIndex(1);
+ lessThanEqualToExpression = new LessThanEqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ Long[] row = { 4751256L };
+ Long[] row1 = { 48512586L };
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Long getLong() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 4751256L;
+ } else {
+ return 48512586L;
+ }
+ }
+ };
+
+ ExpressionResult result = lessThanEqualToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForLessThanEqualToExpressionWithDecimalDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("right_contact", DataType.DECIMAL);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("left_contact", DataType.DECIMAL);
+ left.setColIndex(1);
+ lessThanEqualToExpression = new LessThanEqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ Decimal[] row = new Decimal[] { Decimal.apply(46851.2) };
+ Decimal[] row1 = new Decimal[] { Decimal.apply(45821.02) };
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public BigDecimal getDecimal() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return new BigDecimal(45821.02);
+ } else {
+ return new BigDecimal(46851.2);
+ }
+ }
+ };
+
+ ExpressionResult result = lessThanEqualToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test(expected = FilterUnsupportedException.class) public void testForLessThanEqualToExpressionWithDefaultCase()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("contact", DataType.BOOLEAN);
+ right.setColIndex(0);
+ lessThanEqualToExpression = new LessThanEqualToExpression(right, right);
+ RowImpl value = new RowImpl();
+ Boolean[] row = { true };
+ Object objectRow[] = { row };
+ value.setValues(objectRow);
+ lessThanEqualToExpression.evaluate(value);
+ }
+
+ @Test public void testEvaluateForLessThanEqualToExpressionWithIsNullReturnTrue()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
+ right.setColIndex(0);
+ lessThanEqualToExpression = new LessThanEqualToExpression(right, right);
+ RowImpl value = new RowImpl();
+ Short[] row = { 15856 };
+ Object objectRow[] = { row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public boolean isNull() {
+ return true;
+ }
+ };
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Short getShort() {
+ return 15856;
+ }
+ };
+
+ ExpressionResult result = lessThanEqualToExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+
+ }
+
+ @Test public void testEvaluateForLessThanEqualToExpressionWithLeftAndRightDifferentDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("name", DataType.STRING);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("number", DataType.INT);
+ right.setColIndex(1);
+ lessThanEqualToExpression = new LessThanEqualToExpression(left, right);
+ RowImpl value = new RowImpl();
+ String[] row = { "S" };
+ Integer[] row1 = { 1450 };
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Integer getInt() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 84;
+ } else {
+ return 1450;
+
+ }
+
+ }
+ };
+
+ ExpressionResult result = lessThanEqualToExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testForLessThanEqualToExpressionWithGetString() throws Exception {
+ ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
+ left.setColIndex(0);
+ lessThanEqualToExpression = new LessThanEqualToExpression(left, right);
+ String expected_result =
+ "LessThanEqualTo(ColumnExpression(left_name),ColumnExpression(right_name))";
+ String result = lessThanEqualToExpression.getString();
+ assertEquals(expected_result, result);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java
new file mode 100644
index 0000000..d93f5e4
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/LessThanExpressionUnitTest.java
@@ -0,0 +1,364 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.expression.conditional;
+
+import java.math.BigDecimal;
+import java.sql.Timestamp;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.RowImpl;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.apache.spark.sql.types.Decimal;
+import org.junit.Test;
+
+import static junit.framework.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class LessThanExpressionUnitTest {
+ static LessThanExpression lessThanExpression;
+
+ @Test public void testEvaluateForLessThanExpressionWithStringDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
+ right.setColIndex(1);
+ lessThanExpression = new LessThanExpression(left, right);
+ RowImpl value = new RowImpl();
+ String[] row = { "First String Value" };
+ String[] row1 = { "string1" };
+ Object objectRow[] = { row, row1 };
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public String getString() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return "First String Value";
+
+ } else {
+ return "string1";
+
+ }
+
+ }
+ };
+ value.setValues(objectRow);
+ ExpressionResult result = lessThanExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForLessThanExpressionWithShortDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("id", DataType.SHORT);
+ left.setColIndex(1);
+ lessThanExpression = new LessThanExpression(left, right);
+ RowImpl value = new RowImpl();
+ Short[] row = { 7052 };
+ Short[] row1 = { 7450 };
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Short getShort() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 7052;
+
+ } else {
+ return 7450;
+
+ }
+
+ }
+ };
+
+ ExpressionResult result = lessThanExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+
+ }
+
+ @Test public void testEvaluateForLessThanExpressionWithDoubleDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("right_contact", DataType.DOUBLE);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("left_contact", DataType.DOUBLE);
+ left.setColIndex(1);
+ lessThanExpression = new LessThanExpression(left, right);
+ RowImpl value = new RowImpl();
+ Double[] row = { 2087D };
+ Double[] row1 = { 4454D };
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Double getDouble() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 2087D;
+
+ } else {
+ return 4454D;
+
+ }
+
+ }
+ };
+
+ ExpressionResult result = lessThanExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForLessThanExpressionWithIntDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("right_number", DataType.INT);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("left_number", DataType.INT);
+ left.setColIndex(1);
+ lessThanExpression = new LessThanExpression(left, right);
+ RowImpl value = new RowImpl();
+ Integer[] row = { 1550 };
+ Integer[] row1 = { 1420 };
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Integer getInt() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 1420;
+
+ } else {
+ return 1550;
+
+ }
+
+ }
+ };
+
+ ExpressionResult result = lessThanExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForLessThanExpressionWithTimestampDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ try {
+ ColumnExpression left = new ColumnExpression("timestamp", DataType.TIMESTAMP);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("timestamp", DataType.TIMESTAMP);
+ right.setColIndex(1);
+
+ lessThanExpression = new LessThanExpression(left, right);
+
+ RowImpl value = new RowImpl();
+
+ DateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy");
+
+ Date date = dateFormat.parse("23/09/2007");
+ long time = date.getTime();
+ Timestamp[] row = { new Timestamp(time) };
+
+ Date date1 = dateFormat.parse("24/09/2007");
+ long time1 = date1.getTime();
+ Timestamp[] row1 = { new Timestamp(time1) };
+
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Long getTime() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 1190505600L;
+ } else {
+ return 1190592000L;
+ }
+ }
+ };
+
+ ExpressionResult result = lessThanExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ } catch (ParseException e) {
+ System.out.println("Error while parsing " + e.getMessage());
+ }
+ }
+
+ @Test public void testEvaluateForLessThanExpressionWithLongDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("contact", DataType.LONG);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("contact", DataType.LONG);
+ left.setColIndex(1);
+ lessThanExpression = new LessThanExpression(left, right);
+ RowImpl value = new RowImpl();
+ Long[] row = { 14523656L };
+ Long[] row1 = { 12456325L };
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Long getLong() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 12456325L;
+ } else {
+ return 14523656L;
+ }
+ }
+ };
+
+ ExpressionResult result = lessThanExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForLessThanExpressionWithDecimalDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("contact", DataType.DECIMAL);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("contact", DataType.DECIMAL);
+ left.setColIndex(1);
+ lessThanExpression = new LessThanExpression(left, right);
+ RowImpl value = new RowImpl();
+ Decimal[] row = new Decimal[] { Decimal.apply(256324.0) };
+ Decimal[] row1 = new Decimal[] { Decimal.apply(123451245.0) };
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public BigDecimal getDecimal() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return new BigDecimal(256324.0);
+ } else {
+ return new BigDecimal(123451245.0);
+ }
+ }
+ };
+
+ ExpressionResult result = lessThanExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test(expected = FilterUnsupportedException.class) public void testForLessThanExpressionWithDefaultCase()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("contact", DataType.BOOLEAN);
+ right.setColIndex(0);
+ lessThanExpression = new LessThanExpression(right, right);
+ RowImpl value = new RowImpl();
+ Boolean[] row = { true };
+ Object objectRow[] = { row };
+ value.setValues(objectRow);
+ lessThanExpression.evaluate(value);
+ }
+
+ @Test public void testEvaluateForLessThanExpressionWithIsNullReturnTrue()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
+ right.setColIndex(0);
+ lessThanExpression = new LessThanExpression(right, right);
+ RowImpl value = new RowImpl();
+ Short[] row = { 15 };
+ Object objectRow[] = { row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public boolean isNull() {
+ return true;
+ }
+ };
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Short getShort() {
+ return 15;
+ }
+ };
+
+ ExpressionResult result = lessThanExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+
+ }
+
+ @Test public void testEvaluateForLessThanExpressionWithLeftAndRightDifferentDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("name", DataType.STRING);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("number", DataType.INT);
+ right.setColIndex(1);
+ lessThanExpression = new LessThanExpression(left, right);
+ RowImpl value = new RowImpl();
+ String[] row = { "S" };
+ Integer[] row1 = { 1864 };
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Integer getInt() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 84;
+ } else {
+ return 1864;
+
+ }
+
+ }
+ };
+
+ ExpressionResult result = lessThanExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testForLessThanExpressionWithGetString() throws Exception {
+ ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
+ left.setColIndex(0);
+ lessThanExpression = new LessThanExpression(left, right);
+ String expected_result = "LessThan(ColumnExpression(left_name),ColumnExpression(right_name))";
+ String result = lessThanExpression.getString();
+ assertEquals(expected_result, result);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/ListExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/ListExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/ListExpressionUnitTest.java
new file mode 100644
index 0000000..351b0fa
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/ListExpressionUnitTest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.expression.conditional;
+
+import java.util.List;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.Expression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.RowImpl;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.IsEqual.equalTo;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+public class ListExpressionUnitTest {
+
+ static ListExpression listExpression;
+
+ @Test public void test() throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
+ right.setColIndex(1);
+
+ List<Expression> children = new ArrayList<>();
+ children.add(left);
+ children.add(right);
+
+ listExpression = new ListExpression(children);
+ RowImpl value = new RowImpl();
+ String row = "Row is for left";
+ String row1 = "I am row 1";
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+ String expected_value = "Row is for left";
+ ExpressionResult result = listExpression.evaluate(value);
+ assertThat(expected_value, is(equalTo(result.getList().get(0).getString())));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java
new file mode 100644
index 0000000..43a5878
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotEqualsExpressionUnitTest.java
@@ -0,0 +1,406 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.expression.conditional;
+
+import java.math.BigDecimal;
+import java.sql.Timestamp;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.RowImpl;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.apache.spark.sql.types.Decimal;
+import org.junit.Test;
+
+import static junit.framework.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class NotEqualsExpressionUnitTest {
+
+ static NotEqualsExpression notEqualsExpression;
+
+ @Test public void testEvaluateForNotEqualsExpressionWithBothStringISSame()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
+ right.setColIndex(1);
+ notEqualsExpression = new NotEqualsExpression(left, right);
+ RowImpl value = new RowImpl();
+ String[] row = { "string1" };
+ String[] row1 = { "string2" };
+ Object objectRow[] = { row, row1 };
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public String getString() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return "string1";
+
+ } else {
+ return "string2";
+ }
+ }
+ };
+
+ value.setValues(objectRow);
+ ExpressionResult result = notEqualsExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForNotEqualsExpressionWithShortDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+
+ ColumnExpression left = new ColumnExpression("left_id", DataType.SHORT);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_id", DataType.SHORT);
+ right.setColIndex(1);
+ notEqualsExpression = new NotEqualsExpression(left, right);
+ RowImpl value = new RowImpl();
+ Short[] row = { 15 };
+ Short[] row1 = { 16 };
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Short getShort() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 15;
+
+ } else {
+ return 16;
+
+ }
+
+ }
+ };
+
+ ExpressionResult result = notEqualsExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+
+ }
+
+ @Test public void testEvaluateForNotEqualsExpressionWithIntDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("right_number", DataType.INT);
+ right.setColIndex(1);
+ ColumnExpression left = new ColumnExpression("left_number", DataType.INT);
+ left.setColIndex(0);
+ notEqualsExpression = new NotEqualsExpression(left, right);
+ RowImpl value = new RowImpl();
+ Integer[] row = { 15 };
+ Integer[] row1 = { 16 };
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Integer getInt() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 15;
+
+ } else {
+ return 16;
+ }
+ }
+ };
+
+ ExpressionResult result = notEqualsExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForNotEqualsExpressionWithDoubleDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("right_contact", DataType.DOUBLE);
+ right.setColIndex(1);
+ ColumnExpression left = new ColumnExpression("left_contact", DataType.DOUBLE);
+ left.setColIndex(0);
+ notEqualsExpression = new NotEqualsExpression(left, right);
+ RowImpl value = new RowImpl();
+ Double[] row = { 445.2D };
+ Double[] row1 = { 452.08D };
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Double getDouble() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 445.2D;
+
+ } else {
+ return 452.08D;
+ }
+ }
+ };
+
+ ExpressionResult result = notEqualsExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForNotEqualsExpressionWithLongDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("right_contact", DataType.LONG);
+ right.setColIndex(1);
+ ColumnExpression left = new ColumnExpression("left_contact", DataType.LONG);
+ left.setColIndex(0);
+ notEqualsExpression = new NotEqualsExpression(left, right);
+ RowImpl value = new RowImpl();
+ Long[] row = { 1234567654321L };
+ Long[] row1 = { 12345676541L };
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Long getLong() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 1234567654321L;
+ } else {
+ return 12345676541L;
+ }
+ }
+ };
+
+ ExpressionResult result = notEqualsExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForNotEqualsExpressionWithTimestampDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ try {
+ ColumnExpression left = new ColumnExpression("left_timestamp", DataType.TIMESTAMP);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_timestamp", DataType.TIMESTAMP);
+ right.setColIndex(1);
+
+ notEqualsExpression = new NotEqualsExpression(left, right);
+
+ RowImpl value = new RowImpl();
+
+ DateFormat dateFormat = new SimpleDateFormat("dd/MM/yyyy");
+
+ Date date = dateFormat.parse("23/09/2007");
+ long time = date.getTime();
+ Timestamp[] row = { new Timestamp(time) };
+
+ Date date1 = dateFormat.parse("24/09/2007");
+ long time1 = date1.getTime();
+ Timestamp[] row1 = { new Timestamp(time1) };
+
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Long getTime() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 1190592000L;
+ } else {
+ return 1190505600L;
+ }
+ }
+ };
+
+ ExpressionResult result = notEqualsExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ } catch (ParseException e) {
+ System.out.println("Error while parsing " + e.getMessage());
+ }
+ }
+
+ @Test(expected = FilterUnsupportedException.class) public void testForNotEqualsExpressionWithDefaultCase()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("contact", DataType.BOOLEAN);
+ right.setColIndex(0);
+ notEqualsExpression = new NotEqualsExpression(right, right);
+ RowImpl value = new RowImpl();
+ Boolean[] row = { true };
+ Object objectRow[] = { row };
+ value.setValues(objectRow);
+ notEqualsExpression.evaluate(value);
+ }
+
+ @Test public void testEvaluateWithForNotEqualsExpressionDecimalDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("contact", DataType.DECIMAL);
+ right.setColIndex(1);
+ ColumnExpression left = new ColumnExpression("contact", DataType.DECIMAL);
+ left.setColIndex(0);
+ notEqualsExpression = new NotEqualsExpression(left, right);
+ RowImpl value = new RowImpl();
+ Decimal[] row = new Decimal[] { Decimal.apply(12345.0) };
+ Decimal[] row1 = new Decimal[] { Decimal.apply(1235445.0) };
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public BigDecimal getDecimal() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return new BigDecimal(12345.0);
+ } else {
+ return new BigDecimal(1235445.0);
+ }
+ }
+ };
+
+ ExpressionResult result = notEqualsExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForNotEqualsExpressionWithIsNullReturnTrue()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
+ right.setColIndex(0);
+ notEqualsExpression = new NotEqualsExpression(right, right);
+ RowImpl value = new RowImpl();
+ Short[] row = { 150 };
+ Object objectRow[] = { row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public boolean isNull() {
+ return true;
+ }
+ };
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Short getShort() {
+ return 150;
+ }
+ };
+
+ ExpressionResult result = notEqualsExpression.evaluate(value);
+ assertFalse(result.getBoolean());
+
+ }
+
+ @Test public void testEvaluateForNotEqualsExpressionWithLeftAndRightDifferentDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("name", DataType.STRING);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("number", DataType.INT);
+ right.setColIndex(1);
+ notEqualsExpression = new NotEqualsExpression(left, right);
+ RowImpl value = new RowImpl();
+ String[] row1 = { "S" };
+ Integer[] row = { 14 };
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ Boolean returnMockFlag = true;
+
+ @Mock public Integer getInt() {
+ if (returnMockFlag) {
+ returnMockFlag = false;
+ return 84;
+ } else {
+ return 14;
+ }
+ }
+ };
+
+ ExpressionResult result = notEqualsExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testForNotEqualsExpressionWithGetString() throws Exception {
+ ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
+ ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
+ notEqualsExpression = new NotEqualsExpression(left, right);
+ String expected_result = "NotEquals(ColumnExpression(left_name),ColumnExpression(right_name))";
+ String result = notEqualsExpression.getString();
+ assertEquals(expected_result, result);
+ }
+
+ @Test public void testEvaluateForNotEqualsExpressionWithNullWhileCreatingObject()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
+ right.setColIndex(0);
+ notEqualsExpression = new NotEqualsExpression(right, right, false);
+ RowImpl value = new RowImpl();
+ Short[] row = { 15 };
+ Object objectRow[] = { row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public boolean isNull() {
+ return true;
+ }
+ };
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Short getShort() {
+ return 15;
+ }
+ };
+
+ ExpressionResult result = notEqualsExpression.evaluate(value);
+ assertEquals(DataType.BOOLEAN, result.getDataType());
+
+ }
+
+ @Test public void testEvaluateForNotEqualsExpressionWithNullISTureWhileCreatingObject()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("id", DataType.SHORT);
+ right.setColIndex(0);
+ notEqualsExpression = new NotEqualsExpression(right, right, true);
+ RowImpl value = new RowImpl();
+ Short[] row = { 15 };
+ Object objectRow[] = { row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public boolean isNull() {
+ return true;
+ }
+ };
+
+ ExpressionResult result = notEqualsExpression.evaluate(value);
+ assertEquals(DataType.BOOLEAN, result.getDataType());
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java
new file mode 100644
index 0000000..d22bc43
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/conditional/NotInExpressionUnitTest.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.scan.expression.conditional;
+
+import java.math.BigDecimal;
+import java.sql.Timestamp;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.RowImpl;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.apache.spark.sql.types.Decimal;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class NotInExpressionUnitTest {
+
+ static NotInExpression notInExpression;
+
+ @Test public void testEvaluateForNotInExpressionWithString()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
+ right.setColIndex(1);
+ notInExpression = new NotInExpression(left, right);
+ RowImpl value = new RowImpl();
+ String row = "Row is for left";
+ String row1 = "I am row 1";
+ Object objectRow[] = { row, row1 };
+
+ new MockUp<ExpressionResult>() {
+
+ @Mock public DataType getDataType() {
+ return DataType.STRING;
+ }
+
+ @Mock public String getString() {
+ return "string1";
+ }
+ };
+
+ value.setValues(objectRow);
+ ExpressionResult result = notInExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForNotInExpressionWithShortDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+
+ ColumnExpression left = new ColumnExpression("left_id", DataType.SHORT);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_id", DataType.SHORT);
+ right.setColIndex(1);
+ notInExpression = new NotInExpression(left, right);
+ RowImpl value = new RowImpl();
+ Short row = 15653;
+ Short row1 = 15582;
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Short getShort() {
+ return 15582;
+ }
+ };
+
+ ExpressionResult result = notInExpression.evaluate(value);
+ assertEquals(result.getDataType(), DataType.BOOLEAN);
+
+ }
+
+ @Test public void testEvaluateForNotInExpressionWithIntDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+
+ ColumnExpression left = new ColumnExpression("left_id", DataType.INT);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_id", DataType.INT);
+ right.setColIndex(1);
+ notInExpression = new NotInExpression(left, right);
+ RowImpl value = new RowImpl();
+ Integer row = 150569;
+ Integer row1 = 15052;
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Integer getInt() {
+ return 15052;
+ }
+ };
+
+ ExpressionResult result = notInExpression.evaluate(value);
+ assertEquals(result.getDataType(), DataType.BOOLEAN);
+
+ }
+
+ @Test public void testEvaluateForNotInExpressionWithDoubleDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("left_contact", DataType.DOUBLE);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_contact", DataType.DOUBLE);
+ right.setColIndex(1);
+ notInExpression = new NotInExpression(left, right);
+ RowImpl value = new RowImpl();
+ Double row = 44521D;
+ Double row1 = 44521.023D;
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Double getDouble() {
+ return 44521.023D;
+ }
+ };
+
+ ExpressionResult result = notInExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForNotInExpressionWithLongDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("left_contact", DataType.LONG);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_contact", DataType.LONG);
+ right.setColIndex(1);
+ notInExpression = new NotInExpression(left, right);
+ RowImpl value = new RowImpl();
+ Long row = 123456256325632L;
+ Long row1 = 156212456245556L;
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Long getLong() {
+ return 156212456245556L;
+ }
+ };
+
+ ExpressionResult result = notInExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test public void testEvaluateForNotInExpressionWithTimestampDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ try {
+ ColumnExpression left = new ColumnExpression("left_timestamp", DataType.TIMESTAMP);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_timestamp", DataType.TIMESTAMP);
+ right.setColIndex(1);
+ notInExpression = new NotInExpression(right, right);
+
+ RowImpl value = new RowImpl();
+ DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd");
+ Date date = dateFormat.parse("2007/03/03");
+
+ long time = date.getTime();
+
+ Timestamp row = new Timestamp(time);
+ Timestamp row1 = new Timestamp(time);
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Long getTime() {
+ return 1172860200000L;
+ }
+ };
+
+ ExpressionResult result = notInExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ } catch (ParseException e) {
+ System.out.println("Error while parsing " + e.getMessage());
+ }
+ }
+
+ @Test public void testEvaluateForNotInExpressionWithDecimalDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("left_contact", DataType.DECIMAL);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_contact", DataType.DECIMAL);
+ right.setColIndex(1);
+ notInExpression = new NotInExpression(left, right);
+ RowImpl value = new RowImpl();
+ Decimal row = Decimal.apply(123452154.0);
+ Decimal row1 = Decimal.apply(1234521215454.0);
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public BigDecimal getDecimal() {
+ return new BigDecimal(1234521215454.0);
+ }
+ };
+
+ ExpressionResult result = notInExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+ @Test(expected = FilterUnsupportedException.class) public void testDefaultCaseForNotInExpression()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression left = new ColumnExpression("contact", DataType.BOOLEAN);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("contact", DataType.BOOLEAN);
+ right.setColIndex(1);
+ notInExpression = new NotInExpression(left, right);
+ RowImpl value = new RowImpl();
+ Boolean row = true;
+ Boolean row1 = true;
+ Object objectRow[] = { row, row1 };
+ value.setValues(objectRow);
+ notInExpression.evaluate(value);
+ }
+
+ @Test public void testForNotInExpressionWithGetString() throws Exception {
+ ColumnExpression left = new ColumnExpression("left_name", DataType.STRING);
+ left.setColIndex(0);
+ ColumnExpression right = new ColumnExpression("right_name", DataType.STRING);
+ right.setColIndex(1);
+ notInExpression = new NotInExpression(left, right);
+ String expected_result = "NOT IN(ColumnExpression(left_name),ColumnExpression(right_name))";
+ String result = notInExpression.getString();
+ assertEquals(expected_result, result);
+ }
+ @Test public void testEvaluateForNotInExpressionWithLeftAndRightDifferentDataType()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ ColumnExpression right = new ColumnExpression("name", DataType.STRING);
+ right.setColIndex(0);
+ ColumnExpression left = new ColumnExpression("number", DataType.INT);
+ left.setColIndex(1);
+ notInExpression = new NotInExpression(left, right);
+ RowImpl value = new RowImpl();
+ String row1 = "String1";
+ Integer row = 14523213;
+ Object objectRow[] = { row1, row };
+ value.setValues(objectRow);
+
+ new MockUp<ExpressionResult>() {
+ @Mock public Integer getInt() {
+ return 145232130;
+ }
+ };
+
+ ExpressionResult result = notInExpression.evaluate(value);
+ assertTrue(result.getBoolean());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/AndExpressionTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/AndExpressionTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/AndExpressionTest.java
new file mode 100644
index 0000000..0142ab3
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/AndExpressionTest.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.logical;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.RowImpl;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.junit.Before;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.assertEquals;
+
+public class AndExpressionTest {
+
+ private AndExpression andExpression;
+
+ @Before public void setUp() {
+ ColumnExpression leftExpression = new ColumnExpression("IMEI", DataType.BOOLEAN);
+ ColumnExpression rightExpression = new ColumnExpression("IMEI", DataType.BOOLEAN);
+ andExpression = new AndExpression(leftExpression, rightExpression);
+ }
+
+ @Test(expected = Exception.class) public void testEvaluateForDefault()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ RowImpl rowImpl = new RowImpl();
+ rowImpl.setValues(new Boolean[] { true });
+ final ExpressionResult expressionResult = new ExpressionResult(DataType.STRING, "test");
+ new MockUp<ColumnExpression>() {
+ @Mock public ExpressionResult evaluate(RowIntf value)
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ return expressionResult;
+ }
+ };
+ andExpression.evaluate(rowImpl);
+ }
+
+ @Test public void testEvaluate() throws FilterUnsupportedException, FilterIllegalMemberException {
+ RowImpl rowImpl = new RowImpl();
+ rowImpl.setValues(new Boolean[] { false });
+ final ExpressionResult expressionResult = new ExpressionResult(DataType.BOOLEAN, "test");
+ new MockUp<ColumnExpression>() {
+ @Mock public ExpressionResult evaluate(RowIntf value)
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ return expressionResult;
+ }
+ };
+ ExpressionResult actualValue = andExpression.evaluate(rowImpl);
+ assertTrue(actualValue instanceof ExpressionResult);
+ }
+
+ @Test public void testGetString() {
+ String actualValue = andExpression.getString();
+ String expectedValue = "And(ColumnExpression(IMEI),ColumnExpression(IMEI))";
+ assertEquals(expectedValue, actualValue);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/FalseExpressionTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/FalseExpressionTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/FalseExpressionTest.java
new file mode 100644
index 0000000..bba36cb
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/FalseExpressionTest.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.logical;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.RowImpl;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class FalseExpressionTest {
+ private FalseExpression falseExpression;
+
+ @Before public void setUp() {
+ ColumnExpression columnExpression = new ColumnExpression("IMEI", DataType.BOOLEAN);
+ falseExpression = new FalseExpression(columnExpression);
+ }
+
+ @Test public void testEvaluate() throws FilterUnsupportedException, FilterIllegalMemberException {
+ RowImpl rowImpl = new RowImpl();
+ rowImpl.setValues(new Boolean[] { true });
+ ExpressionResult actualValue = falseExpression.evaluate(rowImpl);
+ assertEquals(new ExpressionResult(DataType.BOOLEAN, false), actualValue);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/OrExpressionTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/OrExpressionTest.java b/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/OrExpressionTest.java
new file mode 100644
index 0000000..49eef24
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/scan/expression/logical/OrExpressionTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.scan.expression.logical;
+
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.scan.expression.ColumnExpression;
+import org.apache.carbondata.core.scan.expression.ExpressionResult;
+import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
+import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
+import org.apache.carbondata.core.scan.filter.intf.RowImpl;
+import org.apache.carbondata.core.scan.filter.intf.RowIntf;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.junit.Before;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.assertEquals;
+
+public class OrExpressionTest {
+ private OrExpression orExpression;
+
+ @Before public void setUp() {
+ ColumnExpression leftExpression = new ColumnExpression("IMEI", DataType.BOOLEAN);
+ ColumnExpression rightExpression = new ColumnExpression("IMEI", DataType.BOOLEAN);
+ orExpression = new OrExpression(leftExpression, rightExpression);
+ }
+
+ @Test public void testGetString() {
+ String actualValue = orExpression.getString();
+ String expectedValue = "Or(ColumnExpression(IMEI),ColumnExpression(IMEI))";
+ assertEquals(expectedValue, actualValue);
+ }
+
+ @Test public void testEvaluate() throws FilterIllegalMemberException, FilterUnsupportedException {
+ RowImpl rowImpl = new RowImpl();
+ rowImpl.setValues(new Boolean[] { false });
+ final ExpressionResult expressionResult = new ExpressionResult(DataType.BOOLEAN, "test");
+ new MockUp<ColumnExpression>() {
+ @Mock public ExpressionResult evaluate(RowIntf value) {
+ return expressionResult;
+ }
+ };
+
+ assertTrue(orExpression.evaluate(rowImpl) instanceof ExpressionResult);
+ }
+
+ @Test(expected = Exception.class) public void testEvaluateForDefault()
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ RowImpl rowImpl = new RowImpl();
+ rowImpl.setValues(new Boolean[] { true });
+ final ExpressionResult expressionResult = new ExpressionResult(DataType.STRING, "test");
+ new MockUp<ColumnExpression>() {
+ @Mock public ExpressionResult evaluate(RowIntf value)
+ throws FilterUnsupportedException, FilterIllegalMemberException {
+ return expressionResult;
+ }
+ };
+ orExpression.evaluate(rowImpl);
+ }
+}
[43/57] [abbrv] incubator-carbondata git commit: move load
Posted by ja...@apache.org.
move load
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/ae17158a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/ae17158a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/ae17158a
Branch: refs/heads/movecorepackage
Commit: ae17158a4cf9390ddd57855e36ed2d99a74cc6ec
Parents: d0d25c9
Author: jackylk <ja...@huawei.com>
Authored: Fri Jan 13 15:32:59 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Fri Jan 13 15:32:59 2017 +0800
----------------------------------------------------------------------
.../carbondata/core/cache/CacheProvider.java | 2 +-
.../dictionary/AbstractDictionaryCache.java | 4 +-
.../cache/dictionary/ColumnDictionaryInfo.java | 2 +-
.../dictionary/DictionaryCacheLoaderImpl.java | 4 +-
.../decimal/CompressionMaxMinInt.java | 4 +-
.../decimal/CompressionMaxMinLong.java | 4 +-
.../decimal/CompressionMaxMinShort.java | 4 +-
.../nondecimal/CompressionNonDecimalByte.java | 2 +-
.../CompressionNonDecimalDefault.java | 4 +-
.../nondecimal/CompressionNonDecimalInt.java | 4 +-
.../nondecimal/CompressionNonDecimalLong.java | 4 +-
.../CompressionNonDecimalMaxMinDefault.java | 4 +-
.../CompressionNonDecimalMaxMinLong.java | 4 +-
.../CompressionNonDecimalMaxMinShort.java | 4 +-
.../compression/none/CompressionNoneByte.java | 4 +-
.../none/CompressionNoneDefault.java | 4 +-
.../compression/none/CompressionNoneInt.java | 4 +-
.../compression/none/CompressionNoneLong.java | 4 +-
.../impl/CompressedDataMeasureDataWrapper.java | 2 +-
...ractHeavyCompressedDoubleArrayDataStore.java | 2 +-
.../core/datastore/BlockIndexStore.java | 8 +-
.../carbondata/core/datastore/DataRefNode.java | 4 +-
.../core/datastore/SegmentTaskIndexStore.java | 2 +-
.../datastore/TableSegmentUniqueIdentifier.java | 4 +-
.../core/datastore/block/BlockInfo.java | 2 +-
.../core/datastore/block/SegmentProperties.java | 13 +-
.../core/datastore/block/TableBlockInfo.java | 4 +-
.../block/TableBlockUniqueIdentifier.java | 2 +-
.../datastore/chunk/MeasureColumnDataChunk.java | 2 +-
.../impl/FixedLengthDimensionDataChunk.java | 5 +-
.../impl/VariableLengthDimensionDataChunk.java | 5 +-
.../chunk/reader/CarbonDataReaderFactory.java | 12 +-
.../reader/DimensionColumnChunkReader.java | 2 +-
.../chunk/reader/MeasureColumnChunkReader.java | 2 +-
.../reader/dimension/AbstractChunkReader.java | 2 +-
...mpressedDimensionChunkFileBasedReaderV1.java | 6 +-
...mpressedDimensionChunkFileBasedReaderV2.java | 4 +-
...CompressedMeasureChunkFileBasedReaderV1.java | 8 +-
...CompressedMeasureChunkFileBasedReaderV2.java | 8 +-
.../chunk/store/DimensionChunkStoreFactory.java | 12 +-
.../chunk/store/MeasureChunkStoreFactory.java | 2 +-
.../UnsafeAbstractDimensionDataChunkStore.java | 2 +-
.../impl/btree/AbstractBTreeBuilder.java | 2 +-
.../impl/btree/AbstractBTreeLeafNode.java | 4 +-
.../datastore/impl/btree/BTreeNonLeafNode.java | 6 +-
.../datastore/impl/btree/BlockBTreeBuilder.java | 2 +-
.../impl/btree/BlockletBTreeBuilder.java | 2 +-
.../impl/btree/BlockletBTreeLeafNode.java | 2 +-
.../dictionary/client/DictionaryClient.java | 4 +-
.../client/DictionaryClientHandler.java | 6 +-
.../IncrementalColumnDictionaryGenerator.java | 16 +-
.../generator/ServerDictionaryGenerator.java | 6 +-
.../generator/TableDictionaryGenerator.java | 6 +-
.../server/DictionaryServerHandler.java | 6 +-
.../DirectDictionaryKeyGeneratorFactory.java | 8 +-
.../DateDirectDictionaryGenerator.java | 2 +-
.../TimeStampDirectDictionaryGenerator.java | 14 +-
.../carbondata/core/load/BlockDetails.java | 87 -----
.../core/load/LoadMetadataDetails.java | 344 -------------------
.../core/memory/HeapMemoryAllocator.java | 2 +-
.../core/metadata/blocklet/DataFileFooter.java | 2 +-
.../core/metadata/schema/table/CarbonTable.java | 10 +-
.../core/metadata/schema/table/TableSchema.java | 2 +-
.../carbondata/core/path/CarbonTablePath.java | 5 +-
.../reader/CarbonDeleteDeltaFileReaderImpl.java | 10 +-
.../reader/CarbonDeleteFilesDataReader.java | 6 +-
.../CarbonDictionaryMetadataReaderImpl.java | 4 +-
.../core/reader/CarbonDictionaryReaderImpl.java | 2 +-
.../CarbonDictionarySortIndexReaderImpl.java | 4 +-
.../impl/AbstractScannedResultCollector.java | 2 +-
.../impl/DictionaryBasedResultCollector.java | 10 +-
.../DictionaryBasedVectorResultCollector.java | 2 +-
.../collector/impl/RawBasedResultCollector.java | 2 +-
.../scan/complextypes/PrimitiveQueryType.java | 2 +-
.../executor/impl/AbstractQueryExecutor.java | 16 +-
.../executor/impl/QueryExecutorProperties.java | 2 +-
.../scan/executor/infos/BlockExecutionInfo.java | 2 +-
.../core/scan/executor/util/QueryUtil.java | 12 +-
.../scan/executor/util/RestructureUtil.java | 2 +-
.../core/scan/expression/ExpressionResult.java | 4 +-
.../scan/filter/FilterExpressionProcessor.java | 4 +-
.../core/scan/filter/FilterProcessor.java | 2 +-
.../carbondata/core/scan/filter/FilterUtil.java | 16 +-
.../ExcludeColGroupFilterExecuterImpl.java | 2 +-
.../executer/ExcludeFilterExecuterImpl.java | 2 +-
.../IncludeColGroupFilterExecuterImpl.java | 2 +-
.../executer/IncludeFilterExecuterImpl.java | 4 +-
.../executer/RowLevelFilterExecuterImpl.java | 12 +-
.../RowLevelRangeGrtThanFiterExecuterImpl.java | 6 +-
...elRangeGrtrThanEquaToFilterExecuterImpl.java | 6 +-
...velRangeLessThanEqualFilterExecuterImpl.java | 8 +-
.../RowLevelRangeLessThanFiterExecuterImpl.java | 8 +-
.../resolver/ConditionalFilterResolverImpl.java | 5 +-
.../filter/resolver/FilterResolverIntf.java | 2 +-
.../resolver/LogicalFilterResolverImpl.java | 2 +-
.../resolver/RowLevelFilterResolverImpl.java | 2 +-
.../RowLevelRangeFilterResolverImpl.java | 8 +-
.../visitor/CustomTypeDictionaryVisitor.java | 2 +-
.../carbondata/core/scan/model/QueryModel.java | 10 +-
.../processor/AbstractDataBlockIterator.java | 4 +-
.../core/scan/processor/BlocksChunkHolder.java | 2 +-
.../processor/impl/DataBlockIteratorImpl.java | 2 +-
.../core/scan/result/AbstractScannedResult.java | 4 +-
.../AbstractDetailQueryResultIterator.java | 16 +-
.../scan/scanner/AbstractBlockletScanner.java | 10 +-
.../core/scan/scanner/impl/FilterScanner.java | 18 +-
.../scan/scanner/impl/NonFilterScanner.java | 2 +-
.../core/service/CarbonCommonFactory.java | 3 -
.../DriverQueryStatisticsRecorderImpl.java | 4 +-
.../core/update/CarbonUpdateUtil.java | 89 +----
.../data/BlockletDeleteDeltaCacheLoader.java | 2 +-
.../core/updatestatus/LoadMetadataDetails.java | 344 +++++++++++++++++++
.../core/updatestatus/SegmentStatusManager.java | 22 +-
.../SegmentUpdateStatusManager.java | 24 +-
.../updatestatus/locks/CarbonLockFactory.java | 2 +-
.../core/updatestatus/locks/HdfsFileLock.java | 2 +-
.../core/updatestatus/locks/LocalFileLock.java | 2 +-
.../updatestatus/locks/ZooKeeperLocking.java | 2 +-
.../util/AbstractDataFileFooterConverter.java | 10 +-
.../core/util/CarbonMetadataUtil.java | 6 +-
.../carbondata/core/util/CarbonProperties.java | 2 +-
.../core/util/CarbonTimeStatisticsFactory.java | 2 +-
.../apache/carbondata/core/util/CarbonUtil.java | 20 +-
.../core/util/DataFileFooterConverter.java | 6 +-
.../core/util/DataFileFooterConverter2.java | 2 +-
.../carbondata/core/util/DataTypeUtil.java | 2 +-
.../core/writer/CarbonDictionaryWriterImpl.java | 8 +-
.../CarbonDictionarySortIndexWriterImpl.java | 8 +-
.../CarbonDictionarySortInfoPreparator.java | 2 +-
.../sortindex/CarbonDictionarySortModel.java | 2 +-
.../core/load/LoadMetadataDetailsUnitTest.java | 119 -------
.../LoadMetadataDetailsUnitTest.java | 121 +++++++
.../hadoop/test/util/StoreCreator.java | 4 +-
.../dataretention/DataRetentionTestCase.scala | 3 +-
.../carbondata/spark/load/CarbonLoaderUtil.java | 2 +-
.../spark/load/DeleteLoadFolders.java | 2 +-
.../spark/merger/CarbonDataMergerUtil.java | 2 +-
.../carbondata/spark/util/CarbonQueryUtil.java | 2 +-
.../carbondata/spark/util/LoadMetadataUtil.java | 2 +-
.../org/apache/carbondata/spark/KeyVal.scala | 2 +-
.../spark/rdd/CarbonDataLoadRDD.scala | 6 +-
.../spark/rdd/CarbonDeleteLoadByDateRDD.scala | 2 +-
.../spark/rdd/DataManagementFunc.scala | 3 +-
.../spark/rdd/NewCarbonDataLoadRDD.scala | 4 +-
.../execution/command/carbonTableSchema.scala | 3 +-
.../scala/org/apache/spark/util/SparkUtil.scala | 2 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 4 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 4 +-
.../processing/csvreaderstep/BlockDetails.java | 87 +++++
.../processing/csvreaderstep/CsvInput.java | 1 -
.../csvreaderstep/UnivocityCsvParserVo.java | 2 -
.../graphgenerator/GraphGenerator.java | 2 +-
.../processing/model/CarbonLoadModel.java | 2 +-
.../carbondata/test/util/StoreCreator.java | 4 +-
154 files changed, 950 insertions(+), 963 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/cache/CacheProvider.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/CacheProvider.java b/core/src/main/java/org/apache/carbondata/core/cache/CacheProvider.java
index 9ab8ac7..216e483 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/CacheProvider.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/CacheProvider.java
@@ -28,11 +28,11 @@ import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
import org.apache.carbondata.core.cache.dictionary.ForwardDictionaryCache;
import org.apache.carbondata.core.cache.dictionary.ReverseDictionaryCache;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.BlockIndexStore;
import org.apache.carbondata.core.datastore.SegmentTaskIndexStore;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.block.TableBlockUniqueIdentifier;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.util.CarbonProperties;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
index b1bf459..5fd06b4 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
@@ -22,16 +22,16 @@ package org.apache.carbondata.core.cache.dictionary;
import java.io.IOException;
import java.util.List;
-import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.core.cache.Cache;
import org.apache.carbondata.core.cache.CacheType;
import org.apache.carbondata.core.cache.CarbonLRUCache;
-import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
import org.apache.carbondata.core.datastorage.impl.FileFactory;
+import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk;
import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader;
+import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.core.service.DictionaryService;
import org.apache.carbondata.core.service.PathService;
import org.apache.carbondata.core.util.CarbonProperties;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
index b08870a..18e88a8 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
@@ -26,8 +26,8 @@ import java.util.Date;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
-import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.util.ByteUtil;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
index c60b911..6e2d899 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
@@ -24,12 +24,12 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import org.apache.carbondata.core.service.CarbonCommonFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.ColumnIdentifier;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.reader.CarbonDictionaryReader;
import org.apache.carbondata.core.reader.sortindex.CarbonDictionarySortIndexReader;
+import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.core.service.DictionaryService;
import org.apache.carbondata.core.util.CarbonUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinInt.java
index 9387155..fcafb0f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinInt.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinInt.java
@@ -25,10 +25,10 @@ import java.nio.ByteBuffer;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.util.ValueCompressionUtil;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinLong.java
index d732e18..3be00e4 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinLong.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinLong.java
@@ -25,10 +25,10 @@ import java.nio.ByteBuffer;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.util.ValueCompressionUtil;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinShort.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinShort.java
index 3f93978..546f903 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinShort.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/decimal/CompressionMaxMinShort.java
@@ -25,10 +25,10 @@ import java.nio.ByteBuffer;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.util.ValueCompressionUtil;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalByte.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalByte.java
index 1767298..7aec7fc 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalByte.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalByte.java
@@ -25,9 +25,9 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastorage.compression.Compressor;
import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
+import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
-import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
public class CompressionNonDecimalByte extends ValueCompressionHolder<byte[]> {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalDefault.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalDefault.java
index ac0ec78..7a3ec18 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalDefault.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalDefault.java
@@ -25,10 +25,10 @@ import java.nio.ByteBuffer;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.util.ValueCompressionUtil;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalInt.java
index 1e5689d..2968702 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalInt.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalInt.java
@@ -25,10 +25,10 @@ import java.nio.ByteBuffer;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.util.ValueCompressionUtil;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalLong.java
index d84b8f2..0083736 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalLong.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalLong.java
@@ -25,10 +25,10 @@ import java.nio.ByteBuffer;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.util.ValueCompressionUtil;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java
index e298c37..e8c386f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinDefault.java
@@ -25,10 +25,10 @@ import java.nio.ByteBuffer;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.util.ValueCompressionUtil;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinLong.java
index 8524c20..ddffc2b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinLong.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinLong.java
@@ -24,11 +24,11 @@ import java.nio.ByteBuffer;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastorage.compression.Compressor;
import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.util.ValueCompressionUtil;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinShort.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinShort.java
index dac557c..1306b1f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinShort.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/nondecimal/CompressionNonDecimalMaxMinShort.java
@@ -25,10 +25,10 @@ import java.nio.ByteBuffer;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.util.ValueCompressionUtil;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneByte.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneByte.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneByte.java
index 308ec8b..32d3e63 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneByte.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneByte.java
@@ -24,10 +24,10 @@ import java.math.BigDecimal;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
public class CompressionNoneByte extends ValueCompressionHolder<byte[]> {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneDefault.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneDefault.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneDefault.java
index 65fa3e5..dad6bdb 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneDefault.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneDefault.java
@@ -25,10 +25,10 @@ import java.nio.ByteBuffer;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.util.ValueCompressionUtil;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneInt.java
index 29a60cf..13ecd8c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneInt.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneInt.java
@@ -25,10 +25,10 @@ import java.nio.ByteBuffer;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.util.ValueCompressionUtil;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneLong.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneLong.java b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneLong.java
index 834467d..c22464d 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneLong.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/compression/none/CompressionNoneLong.java
@@ -25,10 +25,10 @@ import java.nio.ByteBuffer;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.datastorage.compression.Compressor;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.MeasureDataChunkStore;
import org.apache.carbondata.core.util.ValueCompressionUtil;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/CompressedDataMeasureDataWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/CompressedDataMeasureDataWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/CompressedDataMeasureDataWrapper.java
index 6512d25..997543b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/CompressedDataMeasureDataWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/CompressedDataMeasureDataWrapper.java
@@ -19,8 +19,8 @@
package org.apache.carbondata.core.datastorage.impl;
-import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder;
import org.apache.carbondata.core.datastorage.MeasureDataWrapper;
+import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder;
public class CompressedDataMeasureDataWrapper implements MeasureDataWrapper {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
index 8573187..e47cf42 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/impl/data/compressed/AbstractHeavyCompressedDoubleArrayDataStore.java
@@ -23,8 +23,8 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.NodeMeasureDataStore;
-import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
+import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder;
import org.apache.carbondata.core.util.ValueCompressionUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java
index 8fe4601..88d8737 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java
@@ -35,8 +35,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.CarbonLRUCache;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.block.BlockIndex;
import org.apache.carbondata.core.datastore.block.BlockInfo;
@@ -44,11 +43,12 @@ import org.apache.carbondata.core.datastore.block.SegmentTaskIndexWrapper;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.block.TableBlockUniqueIdentifier;
import org.apache.carbondata.core.datastore.exception.IndexBuilderException;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.scan.model.QueryModel;
import org.apache.carbondata.core.update.CarbonUpdateUtil;
import org.apache.carbondata.core.update.data.UpdateVO;
import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.scan.model.QueryModel;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
index 219ee73..9d4287a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/DataRefNode.java
@@ -20,10 +20,10 @@ package org.apache.carbondata.core.datastore;
import java.io.IOException;
-import org.apache.carbondata.core.update.data.BlockletLevelDeleteDeltaDataCache;
+import org.apache.carbondata.core.datastorage.FileHolder;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
-import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.update.data.BlockletLevelDeleteDeltaDataCache;
/**
* Interface data block reference
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
index 8feae0c..21be5e8 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStore.java
@@ -32,12 +32,12 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.Cache;
import org.apache.carbondata.core.cache.CarbonLRUCache;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.block.SegmentTaskIndex;
import org.apache.carbondata.core.datastore.block.SegmentTaskIndexWrapper;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.exception.IndexBuilderException;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.path.CarbonTablePath.DataFileUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/TableSegmentUniqueIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/TableSegmentUniqueIdentifier.java b/core/src/main/java/org/apache/carbondata/core/datastore/TableSegmentUniqueIdentifier.java
index 71b7c3f..30407b4 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/TableSegmentUniqueIdentifier.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/TableSegmentUniqueIdentifier.java
@@ -22,10 +22,10 @@ package org.apache.carbondata.core.datastore;
import java.util.List;
import java.util.Map;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
-import org.apache.carbondata.core.datastore.block.TableBlockInfo;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
/**
* Class holds the absoluteTableIdentifier and segmentId to uniquely identify a segment
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/block/BlockInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/BlockInfo.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/BlockInfo.java
index 37382f3..dcb0a3e 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/BlockInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/BlockInfo.java
@@ -18,8 +18,8 @@
*/
package org.apache.carbondata.core.datastore.block;
-import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.path.CarbonTablePath;
/**
* Below class will be used to store table block info
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
index e84c8ba..dde8e76 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentProperties.java
@@ -28,17 +28,18 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-import org.apache.carbondata.core.metadata.DataType;
-import org.apache.carbondata.core.metadata.Encoding;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.columnar.ColumnGroupModel;
import org.apache.carbondata.core.keygenerator.KeyGenerator;
import org.apache.carbondata.core.keygenerator.columnar.ColumnarSplitter;
-import org.apache.carbondata.core.keygenerator.columnar.impl.MultiDimKeyVarLengthVariableSplitGenerator;
+import org.apache.carbondata.core.keygenerator.columnar.impl
+ .MultiDimKeyVarLengthVariableSplitGenerator;
import org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator;
+import org.apache.carbondata.core.metadata.DataType;
+import org.apache.carbondata.core.metadata.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.commons.lang3.ArrayUtils;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
index 4f20ceb..c7ce3db 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
@@ -22,11 +22,11 @@ import java.io.Serializable;
import java.util.HashMap;
import java.util.Map;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.path.CarbonTablePath.DataFileUtil;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
/**
* class will be used to pass the block detail detail will be passed form driver
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockUniqueIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockUniqueIdentifier.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockUniqueIdentifier.java
index f2fbba7..4a05ff8 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockUniqueIdentifier.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockUniqueIdentifier.java
@@ -1,8 +1,8 @@
package org.apache.carbondata.core.datastore.block;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
/**
* Class : Holds the info to uniquely identify a blocks
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java
index d2ee1a7..ce09305 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/MeasureColumnDataChunk.java
@@ -18,8 +18,8 @@
*/
package org.apache.carbondata.core.datastore.chunk;
-import org.apache.carbondata.core.metadata.blocklet.datachunk.PresenceMeta;
import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.metadata.blocklet.datachunk.PresenceMeta;
/**
* Holder for measure column chunk
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
index 9eee078..6edc977 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
@@ -18,9 +18,10 @@
*/
package org.apache.carbondata.core.datastore.chunk.impl;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory
+ .DimensionStoreType;
import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
index 7c16f54..549d6d8 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
@@ -20,9 +20,10 @@ package org.apache.carbondata.core.datastore.chunk.impl;
import java.util.Arrays;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory.DimensionStoreType;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionChunkStoreFactory
+ .DimensionStoreType;
import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
import org.apache.carbondata.core.scan.result.vector.CarbonColumnVector;
import org.apache.carbondata.core.scan.result.vector.ColumnVectorInfo;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
index 5ac5702..c83f2d7 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/CarbonDataReaderFactory.java
@@ -18,11 +18,15 @@
*/
package org.apache.carbondata.core.datastore.chunk.reader;
+import org.apache.carbondata.core.datastore.chunk.reader.dimension
+ .v1.CompressedDimensionChunkFileBasedReaderV1;
+import org.apache.carbondata.core.datastore.chunk.reader.dimension
+ .v2.CompressedDimensionChunkFileBasedReaderV2;
+import org.apache.carbondata.core.datastore.chunk.reader.measure
+ .v1.CompressedMeasureChunkFileBasedReaderV1;
+import org.apache.carbondata.core.datastore.chunk.reader.measure
+ .v2.CompressedMeasureChunkFileBasedReaderV2;
import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
-import org.apache.carbondata.core.datastore.chunk.reader.dimension.v1.CompressedDimensionChunkFileBasedReaderV1;
-import org.apache.carbondata.core.datastore.chunk.reader.dimension.v2.CompressedDimensionChunkFileBasedReaderV2;
-import org.apache.carbondata.core.datastore.chunk.reader.measure.v1.CompressedMeasureChunkFileBasedReaderV1;
-import org.apache.carbondata.core.datastore.chunk.reader.measure.v2.CompressedMeasureChunkFileBasedReaderV2;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
index aeb7ff9..92dcb60 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/DimensionColumnChunkReader.java
@@ -20,8 +20,8 @@ package org.apache.carbondata.core.datastore.chunk.reader;
import java.io.IOException;
-import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
/**
* Interface for reading the data chunk
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
index ec86612..94004d9 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/MeasureColumnChunkReader.java
@@ -20,8 +20,8 @@ package org.apache.carbondata.core.datastore.chunk.reader;
import java.io.IOException;
-import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
/**
* Reader interface for reading the measure blocks from file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java
index 4e45bc3..4cc24d9 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/AbstractChunkReader.java
@@ -18,10 +18,10 @@
*/
package org.apache.carbondata.core.datastore.chunk.reader.dimension;
-import org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastorage.compression.Compressor;
import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
+import org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader;
import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
import org.apache.carbondata.core.util.CarbonProperties;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
index 01df68b..dd487d9 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v1/CompressedDimensionChunkFileBasedReaderV1.java
@@ -21,16 +21,16 @@ package org.apache.carbondata.core.datastore.chunk.reader.dimension.v1;
import java.io.IOException;
import java.util.List;
+import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.datastorage.columnar.UnBlockIndexer;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.ColumnGroupDimensionDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
import org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReader;
+import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk;
-import org.apache.carbondata.core.metadata.Encoding;
-import org.apache.carbondata.core.datastorage.FileHolder;
-import org.apache.carbondata.core.datastorage.columnar.UnBlockIndexer;
import org.apache.carbondata.core.util.CarbonUtil;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
index 48b5aeb..3bddf4e 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/dimension/v2/CompressedDimensionChunkFileBasedReaderV2.java
@@ -21,14 +21,14 @@ package org.apache.carbondata.core.datastore.chunk.reader.dimension.v2;
import java.io.IOException;
import java.util.List;
+import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.datastorage.columnar.UnBlockIndexer;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.ColumnGroupDimensionDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
import org.apache.carbondata.core.datastore.chunk.impl.VariableLengthDimensionDataChunk;
import org.apache.carbondata.core.datastore.chunk.reader.dimension.AbstractChunkReader;
import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
-import org.apache.carbondata.core.datastorage.FileHolder;
-import org.apache.carbondata.core.datastorage.columnar.UnBlockIndexer;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.format.DataChunk2;
import org.apache.carbondata.format.Encoding;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
index 562ea8c..48cb244 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v1/CompressedMeasureChunkFileBasedReaderV1.java
@@ -21,15 +21,15 @@ package org.apache.carbondata.core.datastore.chunk.reader.measure.v1;
import java.io.IOException;
import java.util.List;
-import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReader;
-import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
-import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk;
import org.apache.carbondata.core.datastorage.FileHolder;
import org.apache.carbondata.core.datastorage.compression.ReaderCompressModel;
import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReader;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
+import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
+import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk;
import org.apache.carbondata.core.util.ValueCompressionUtil;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
index 90fa6f3..3325299 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
@@ -23,16 +23,16 @@ import java.util.ArrayList;
import java.util.BitSet;
import java.util.List;
-import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
-import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReader;
-import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
-import org.apache.carbondata.core.metadata.blocklet.datachunk.PresenceMeta;
import org.apache.carbondata.core.datastorage.FileHolder;
import org.apache.carbondata.core.datastorage.compression.CompressorFactory;
import org.apache.carbondata.core.datastorage.compression.ValueCompressionHolder;
import org.apache.carbondata.core.datastorage.compression.WriterCompressModel;
import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder;
+import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
+import org.apache.carbondata.core.datastore.chunk.reader.measure.AbstractMeasureChunkReader;
import org.apache.carbondata.core.metadata.ValueEncoderMeta;
+import org.apache.carbondata.core.metadata.blocklet.BlockletInfo;
+import org.apache.carbondata.core.metadata.blocklet.datachunk.PresenceMeta;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.format.DataChunk2;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java
index f667e65..df054c5 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/DimensionChunkStoreFactory.java
@@ -16,11 +16,15 @@
*/
package org.apache.carbondata.core.datastore.chunk.store;
-import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeFixedLengthDimensionDataChunkStore;
-import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeVariableLengthDimensionDataChunkStore;
-import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeFixedLengthDimensionDataChunkStore;
-import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeVariableLengthDimesionDataChunkStore;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.chunk.store.impl.safe
+ .SafeFixedLengthDimensionDataChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.safe
+ .SafeVariableLengthDimensionDataChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe
+ .UnsafeFixedLengthDimensionDataChunkStore;
+import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe
+ .UnsafeVariableLengthDimesionDataChunkStore;
import org.apache.carbondata.core.util.CarbonProperties;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/MeasureChunkStoreFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/MeasureChunkStoreFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/MeasureChunkStoreFactory.java
index 8a87e49..a95ae11 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/MeasureChunkStoreFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/MeasureChunkStoreFactory.java
@@ -16,6 +16,7 @@
*/
package org.apache.carbondata.core.datastore.chunk.store;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeByteMeasureChunkStore;
import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeDoubleMeasureChunkStore;
import org.apache.carbondata.core.datastore.chunk.store.impl.safe.SafeIntMeasureChunkStore;
@@ -26,7 +27,6 @@ import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeDouble
import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeIntMeasureChunkStore;
import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeLongMeasureChunkStore;
import org.apache.carbondata.core.datastore.chunk.store.impl.unsafe.UnsafeShortMeasureChunkStore;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
index 718170c..4c5c05c 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
@@ -18,8 +18,8 @@
*/
package org.apache.carbondata.core.datastore.chunk.store.impl.unsafe;
-import org.apache.carbondata.core.datastore.chunk.store.DimensionDataChunkStore;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastore.chunk.store.DimensionDataChunkStore;
import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
import org.apache.carbondata.core.memory.MemoryBlock;
import org.apache.carbondata.core.unsafe.CarbonUnsafe;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeBuilder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeBuilder.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeBuilder.java
index 2ea58b5..6b64410 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeBuilder.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeBuilder.java
@@ -22,9 +22,9 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.BtreeBuilder;
import org.apache.carbondata.core.datastore.IndexKey;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.util.CarbonProperties;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
index 14df51f..4907dd9 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/AbstractBTreeLeafNode.java
@@ -20,12 +20,12 @@ package org.apache.carbondata.core.datastore.impl.btree;
import java.io.IOException;
-import org.apache.carbondata.core.update.data.BlockletLevelDeleteDeltaDataCache;
+import org.apache.carbondata.core.datastorage.FileHolder;
import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
-import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.update.data.BlockletLevelDeleteDeltaDataCache;
/**
* Non leaf node abstract class
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
index c2ffae9..3176b9b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BTreeNonLeafNode.java
@@ -21,13 +21,13 @@ package org.apache.carbondata.core.datastore.impl.btree;
import java.util.ArrayList;
import java.util.List;
-import org.apache.carbondata.core.update.data.BlockletLevelDeleteDeltaDataCache;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastorage.FileHolder;
import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.FileHolder;
+import org.apache.carbondata.core.update.data.BlockletLevelDeleteDeltaDataCache;
/**
* No leaf node of a b+tree class which will keep the matadata(start key) of the
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockBTreeBuilder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockBTreeBuilder.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockBTreeBuilder.java
index 0d20002..60fb89f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockBTreeBuilder.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockBTreeBuilder.java
@@ -41,9 +41,9 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.BTreeBuilderInfo;
import org.apache.carbondata.core.datastore.IndexKey;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
/**
* Below class will be used to build the btree BTree will be built for all the
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeBuilder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeBuilder.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeBuilder.java
index 4d11014..6a37706 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeBuilder.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeBuilder.java
@@ -23,9 +23,9 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.BTreeBuilderInfo;
import org.apache.carbondata.core.datastore.IndexKey;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
/**
* Btree based builder which will build the leaf node in a b+ tree format
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
index f3d2bef..4951c95 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/btree/BlockletBTreeLeafNode.java
@@ -20,6 +20,7 @@ package org.apache.carbondata.core.datastore.impl.btree;
import java.io.IOException;
+import org.apache.carbondata.core.datastorage.FileHolder;
import org.apache.carbondata.core.datastore.BTreeBuilderInfo;
import org.apache.carbondata.core.datastore.chunk.DimensionColumnDataChunk;
import org.apache.carbondata.core.datastore.chunk.MeasureColumnDataChunk;
@@ -27,7 +28,6 @@ import org.apache.carbondata.core.datastore.chunk.reader.CarbonDataReaderFactory
import org.apache.carbondata.core.datastore.chunk.reader.DimensionColumnChunkReader;
import org.apache.carbondata.core.datastore.chunk.reader.MeasureColumnChunkReader;
import org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex;
-import org.apache.carbondata.core.datastorage.FileHolder;
/**
* Leaf node class of a Blocklet btree
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClient.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClient.java b/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClient.java
index 92bc56a..802def5 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClient.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClient.java
@@ -27,7 +27,9 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.dictionary.generator.key.DictionaryKey;
import org.jboss.netty.bootstrap.ClientBootstrap;
-import org.jboss.netty.channel.*;
+import org.jboss.netty.channel.ChannelPipeline;
+import org.jboss.netty.channel.ChannelPipelineFactory;
+import org.jboss.netty.channel.Channels;
import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory;
import org.jboss.netty.handler.codec.serialization.ClassResolvers;
import org.jboss.netty.handler.codec.serialization.ObjectDecoder;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java b/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java
index d5ca781..5dba202 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java
@@ -28,7 +28,11 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.dictionary.generator.key.DictionaryKey;
import org.apache.carbondata.core.dictionary.generator.key.KryoRegister;
-import org.jboss.netty.channel.*;
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.channel.ChannelStateEvent;
+import org.jboss.netty.channel.ExceptionEvent;
+import org.jboss.netty.channel.MessageEvent;
+import org.jboss.netty.channel.SimpleChannelHandler;
/**
* Client handler to get data.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java
index b95f2c9..3902746 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/IncrementalColumnDictionaryGenerator.java
@@ -19,10 +19,11 @@
package org.apache.carbondata.core.dictionary.generator;
import java.io.IOException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
-import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.Cache;
@@ -30,15 +31,16 @@ import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.cache.CacheType;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
-import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
-import org.apache.carbondata.core.metadata.ColumnIdentifier;
-import org.apache.carbondata.core.metadata.CarbonMetadata;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.devapi.BiDictionary;
import org.apache.carbondata.core.devapi.DictionaryGenerationException;
import org.apache.carbondata.core.devapi.DictionaryGenerator;
+import org.apache.carbondata.core.metadata.CarbonMetadata;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.core.service.DictionaryService;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.core.util.DataTypeUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ae17158a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGenerator.java
index b522560..0da6f94 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGenerator.java
@@ -21,12 +21,12 @@ package org.apache.carbondata.core.dictionary.generator;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
-import org.apache.carbondata.core.metadata.CarbonMetadata;
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.devapi.DictionaryGenerationException;
import org.apache.carbondata.core.devapi.DictionaryGenerator;
import org.apache.carbondata.core.dictionary.generator.key.DictionaryKey;
+import org.apache.carbondata.core.metadata.CarbonMetadata;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
/**
* This is the dictionary generator for all tables. It generates dictionary
[34/57] [abbrv] incubator-carbondata git commit: move
org.apache.carbon.common in core
Posted by ja...@apache.org.
move org.apache.carbon.common in core
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/952cf517
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/952cf517
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/952cf517
Branch: refs/heads/movecorepackage
Commit: 952cf51745ffeb69a6a5dcc93405364c359933c8
Parents: d2ae472
Author: jackylk <ja...@huawei.com>
Authored: Fri Jan 13 14:43:33 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Fri Jan 13 14:43:33 2017 +0800
----------------------------------------------------------------------
.../common/ext/ColumnUniqueIdGenerator.java | 41 -
.../common/ext/DictionaryFactory.java | 119 --
.../carbondata/common/ext/PathFactory.java | 46 -
.../common/factory/CarbonCommonFactory.java | 54 -
.../BlockletLevelDeleteDeltaDataCache.java | 46 -
.../iudprocessor/iuddata/BlockMappingVO.java | 54 -
.../iuddata/BlockletDeleteDeltaCacheLoader.java | 80 --
.../iuddata/DeleteDeltaCacheLoaderIntf.java | 27 -
.../iudprocessor/iuddata/RowCountDetailsVO.java | 68 -
.../core/AbsoluteTableIdentifier.java | 2 +-
.../dictionary/AbstractDictionaryCache.java | 6 +-
.../dictionary/DictionaryCacheLoaderImpl.java | 2 +-
.../core/compression/BigDecimalCompressor.java | 2 +-
.../core/compression/BigIntCompressor.java | 2 +-
.../core/compression/DoubleCompressor.java | 2 +-
.../core/compression/ValueCompressor.java | 2 +-
.../carbondata/core/datastorage/FileHolder.java | 88 ++
.../core/datastorage/MeasureDataWrapper.java | 30 +
.../core/datastorage/NodeMeasureDataStore.java | 34 +
.../columnar/BlockIndexerStorageForInt.java | 226 +++
.../BlockIndexerStorageForNoInvertedIndex.java | 159 ++
.../datastorage/columnar/ColumnGroupModel.java | 116 ++
.../columnar/ColumnWithIntIndex.java | 82 ++
.../columnar/ColumnWithIntIndexForHighCard.java | 49 +
.../columnar/ColumnarKeyStoreDataHolder.java | 54 +
.../columnar/ColumnarKeyStoreMetadata.java | 53 +
.../core/datastorage/columnar/IndexStorage.java | 44 +
.../datastorage/columnar/UnBlockIndexer.java | 78 +
.../datastorage/compression/Compressor.java | 60 +
.../compression/CompressorFactory.java | 52 +
.../compression/MeasureMetaDataModel.java | 128 ++
.../compression/ReaderCompressModel.java | 61 +
.../compression/SnappyCompressor.java | 222 +++
.../compression/ValueCompressionHolder.java | 115 ++
.../compression/WriterCompressModel.java | 220 +++
.../decimal/CompressionMaxMinByte.java | 115 ++
.../decimal/CompressionMaxMinDefault.java | 117 ++
.../decimal/CompressionMaxMinInt.java | 113 ++
.../decimal/CompressionMaxMinLong.java | 113 ++
.../decimal/CompressionMaxMinShort.java | 115 ++
.../nondecimal/CompressionNonDecimalByte.java | 99 ++
.../CompressionNonDecimalDefault.java | 101 ++
.../nondecimal/CompressionNonDecimalInt.java | 100 ++
.../nondecimal/CompressionNonDecimalLong.java | 101 ++
.../CompressionNonDecimalMaxMinByte.java | 105 ++
.../CompressionNonDecimalMaxMinDefault.java | 107 ++
.../CompressionNonDecimalMaxMinInt.java | 105 ++
.../CompressionNonDecimalMaxMinLong.java | 106 ++
.../CompressionNonDecimalMaxMinShort.java | 104 ++
.../nondecimal/CompressionNonDecimalShort.java | 100 ++
.../compression/none/CompressionNoneByte.java | 104 ++
.../none/CompressionNoneDefault.java | 102 ++
.../compression/none/CompressionNoneInt.java | 102 ++
.../compression/none/CompressionNoneLong.java | 99 ++
.../compression/none/CompressionNoneShort.java | 104 ++
.../compression/type/CompressionBigDecimal.java | 146 ++
.../dataholder/CarbonReadDataHolder.java | 50 +
.../dataholder/CarbonWriteDataHolder.java | 280 ++++
.../filesystem/AbstractDFSCarbonFile.java | 217 +++
.../filesystem/AlluxioCarbonFile.java | 130 ++
.../core/datastorage/filesystem/CarbonFile.java | 66 +
.../filesystem/CarbonFileFilter.java | 24 +
.../datastorage/filesystem/HDFSCarbonFile.java | 128 ++
.../datastorage/filesystem/LocalCarbonFile.java | 229 +++
.../filesystem/ViewFSCarbonFile.java | 126 ++
.../impl/CompressedDataMeasureDataWrapper.java | 37 +
.../datastorage/impl/DFSFileHolderImpl.java | 134 ++
.../core/datastorage/impl/FileFactory.java | 485 +++++++
.../core/datastorage/impl/FileHolderImpl.java | 198 +++
...ractHeavyCompressedDoubleArrayDataStore.java | 90 ++
...yCompressedDoubleArrayDataInMemoryStore.java | 30 +
.../core/datastorage/store/FileHolder.java | 88 --
.../datastorage/store/MeasureDataWrapper.java | 30 -
.../datastorage/store/NodeMeasureDataStore.java | 34 -
.../columnar/BlockIndexerStorageForInt.java | 226 ---
.../BlockIndexerStorageForNoInvertedIndex.java | 159 --
.../store/columnar/ColumnGroupModel.java | 116 --
.../store/columnar/ColumnWithIntIndex.java | 82 --
.../columnar/ColumnWithIntIndexForHighCard.java | 49 -
.../columnar/ColumnarKeyStoreDataHolder.java | 54 -
.../columnar/ColumnarKeyStoreMetadata.java | 53 -
.../store/columnar/IndexStorage.java | 44 -
.../store/columnar/UnBlockIndexer.java | 78 -
.../store/compression/Compressor.java | 60 -
.../store/compression/CompressorFactory.java | 52 -
.../store/compression/MeasureMetaDataModel.java | 128 --
.../store/compression/ReaderCompressModel.java | 61 -
.../store/compression/SnappyCompressor.java | 222 ---
.../compression/ValueCompressionHolder.java | 115 --
.../store/compression/WriterCompressModel.java | 220 ---
.../decimal/CompressionMaxMinByte.java | 115 --
.../decimal/CompressionMaxMinDefault.java | 117 --
.../decimal/CompressionMaxMinInt.java | 113 --
.../decimal/CompressionMaxMinLong.java | 113 --
.../decimal/CompressionMaxMinShort.java | 115 --
.../nondecimal/CompressionNonDecimalByte.java | 99 --
.../CompressionNonDecimalDefault.java | 101 --
.../nondecimal/CompressionNonDecimalInt.java | 100 --
.../nondecimal/CompressionNonDecimalLong.java | 101 --
.../CompressionNonDecimalMaxMinByte.java | 105 --
.../CompressionNonDecimalMaxMinDefault.java | 107 --
.../CompressionNonDecimalMaxMinInt.java | 105 --
.../CompressionNonDecimalMaxMinLong.java | 106 --
.../CompressionNonDecimalMaxMinShort.java | 104 --
.../nondecimal/CompressionNonDecimalShort.java | 100 --
.../compression/none/CompressionNoneByte.java | 104 --
.../none/CompressionNoneDefault.java | 102 --
.../compression/none/CompressionNoneInt.java | 102 --
.../compression/none/CompressionNoneLong.java | 99 --
.../compression/none/CompressionNoneShort.java | 104 --
.../compression/type/CompressionBigDecimal.java | 146 --
.../store/dataholder/CarbonReadDataHolder.java | 50 -
.../store/dataholder/CarbonWriteDataHolder.java | 280 ----
.../store/filesystem/AbstractDFSCarbonFile.java | 217 ---
.../store/filesystem/AlluxioCarbonFile.java | 130 --
.../store/filesystem/CarbonFile.java | 66 -
.../store/filesystem/CarbonFileFilter.java | 24 -
.../store/filesystem/HDFSCarbonFile.java | 128 --
.../store/filesystem/LocalCarbonFile.java | 229 ---
.../store/filesystem/ViewFSCarbonFile.java | 126 --
.../impl/CompressedDataMeasureDataWrapper.java | 37 -
.../store/impl/DFSFileHolderImpl.java | 134 --
.../datastorage/store/impl/FileFactory.java | 485 -------
.../datastorage/store/impl/FileHolderImpl.java | 198 ---
...ractHeavyCompressedDoubleArrayDataStore.java | 90 --
...yCompressedDoubleArrayDataInMemoryStore.java | 30 -
.../core/datastorage/util/StoreFactory.java | 35 -
.../core/datastore/BlockIndexStore.java | 4 +-
.../carbondata/core/datastore/DataRefNode.java | 4 +-
.../core/datastore/SegmentTaskIndexStore.java | 2 +-
.../core/datastore/block/SegmentProperties.java | 2 +-
.../block/SegmentTaskIndexWrapper.java | 2 +-
.../core/datastore/block/TableBlockInfo.java | 2 +-
.../chunk/DimensionColumnDataChunk.java | 4 +-
.../datastore/chunk/MeasureColumnDataChunk.java | 2 +-
.../impl/ColumnGroupDimensionDataChunk.java | 4 +-
.../impl/FixedLengthDimensionDataChunk.java | 6 +-
.../impl/VariableLengthDimensionDataChunk.java | 6 +-
.../reader/DimensionColumnChunkReader.java | 2 +-
.../chunk/reader/MeasureColumnChunkReader.java | 2 +-
.../reader/dimension/AbstractChunkReader.java | 4 +-
...mpressedDimensionChunkFileBasedReaderV1.java | 4 +-
...mpressedDimensionChunkFileBasedReaderV2.java | 4 +-
...CompressedMeasureChunkFileBasedReaderV1.java | 8 +-
...CompressedMeasureChunkFileBasedReaderV2.java | 10 +-
.../impl/btree/AbstractBTreeLeafNode.java | 4 +-
.../datastore/impl/btree/BTreeNonLeafNode.java | 4 +-
.../impl/btree/BlockletBTreeLeafNode.java | 2 +-
.../IncrementalColumnDictionaryGenerator.java | 2 +-
.../carbondata/core/load/BlockDetails.java | 2 +-
.../core/metadata/BlockletInfoColumnar.java | 2 +-
.../carbondata/core/partition/Partitioner.java | 26 -
.../partition/impl/HashPartitionerImpl.java | 105 --
.../carbondata/core/path/CarbonTablePath.java | 6 +-
.../reader/CarbonDeleteDeltaFileReaderImpl.java | 2 +-
.../reader/CarbonDeleteFilesDataReader.java | 2 +-
.../CarbonDictionaryMetadataReaderImpl.java | 2 +-
.../core/reader/CarbonDictionaryReaderImpl.java | 2 +-
.../carbondata/core/reader/ThriftReader.java | 2 +-
.../CarbonDictionarySortIndexReaderImpl.java | 4 +-
.../scan/collector/ScannedResultCollector.java | 45 +
.../impl/AbstractScannedResultCollector.java | 159 ++
.../impl/DictionaryBasedResultCollector.java | 172 +++
.../DictionaryBasedVectorResultCollector.java | 136 ++
.../collector/impl/RawBasedResultCollector.java | 74 +
.../core/scan/complextypes/ArrayQueryType.java | 103 ++
.../scan/complextypes/ComplexQueryType.java | 64 +
.../scan/complextypes/PrimitiveQueryType.java | 118 ++
.../core/scan/complextypes/StructQueryType.java | 114 ++
.../core/scan/executor/QueryExecutor.java | 51 +
.../scan/executor/QueryExecutorFactory.java | 38 +
.../exception/QueryExecutionException.java | 94 ++
.../executor/impl/AbstractQueryExecutor.java | 483 +++++++
.../scan/executor/impl/DetailQueryExecutor.java | 49 +
.../executor/impl/QueryExecutorProperties.java | 73 +
.../impl/VectorDetailQueryExecutor.java | 46 +
.../scan/executor/infos/AggregatorInfo.java | 105 ++
.../scan/executor/infos/BlockExecutionInfo.java | 604 ++++++++
.../scan/executor/infos/KeyStructureInfo.java | 100 ++
.../core/scan/executor/util/QueryUtil.java | 882 ++++++++++++
.../scan/executor/util/RestructureUtil.java | 140 ++
.../core/scan/expression/BinaryExpression.java | 43 +
.../core/scan/expression/ColumnExpression.java | 114 ++
.../core/scan/expression/Expression.java | 49 +
.../core/scan/expression/ExpressionResult.java | 545 +++++++
.../core/scan/expression/LeafExpression.java | 24 +
.../core/scan/expression/LiteralExpression.java | 69 +
.../core/scan/expression/UnknownExpression.java | 28 +
.../BinaryConditionalExpression.java | 37 +
.../conditional/ConditionalExpression.java | 37 +
.../conditional/EqualToExpression.java | 109 ++
.../GreaterThanEqualToExpression.java | 91 ++
.../conditional/GreaterThanExpression.java | 94 ++
.../expression/conditional/InExpression.java | 99 ++
.../conditional/LessThanEqualToExpression.java | 93 ++
.../conditional/LessThanExpression.java | 96 ++
.../expression/conditional/ListExpression.java | 62 +
.../conditional/NotEqualsExpression.java | 105 ++
.../expression/conditional/NotInExpression.java | 98 ++
.../exception/FilterIllegalMemberException.java | 94 ++
.../exception/FilterUnsupportedException.java | 88 ++
.../scan/expression/logical/AndExpression.java | 63 +
.../logical/BinaryLogicalExpression.java | 109 ++
.../expression/logical/FalseExpression.java | 70 +
.../scan/expression/logical/OrExpression.java | 62 +
.../core/scan/filter/DimColumnFilterInfo.java | 80 ++
.../scan/filter/FilterExpressionProcessor.java | 338 +++++
.../core/scan/filter/FilterProcessor.java | 57 +
.../carbondata/core/scan/filter/FilterUtil.java | 1353 ++++++++++++++++++
.../core/scan/filter/GenericQueryType.java | 49 +
.../filter/executer/AndFilterExecuterImpl.java | 63 +
.../executer/DimColumnExecuterFilterInfo.java | 32 +
.../ExcludeColGroupFilterExecuterImpl.java | 135 ++
.../executer/ExcludeFilterExecuterImpl.java | 153 ++
.../scan/filter/executer/FilterExecuter.java | 47 +
.../IncludeColGroupFilterExecuterImpl.java | 207 +++
.../executer/IncludeFilterExecuterImpl.java | 178 +++
.../filter/executer/OrFilterExecuterImpl.java | 53 +
.../executer/RestructureFilterExecuterImpl.java | 54 +
.../executer/RowLevelFilterExecuterImpl.java | 368 +++++
.../RowLevelRangeGrtThanFiterExecuterImpl.java | 212 +++
...elRangeGrtrThanEquaToFilterExecuterImpl.java | 199 +++
...velRangeLessThanEqualFilterExecuterImpl.java | 248 ++++
.../RowLevelRangeLessThanFiterExecuterImpl.java | 255 ++++
.../RowLevelRangeTypeExecuterFacory.java | 92 ++
.../core/scan/filter/intf/ExpressionType.java | 45 +
.../scan/filter/intf/FilterExecuterType.java | 28 +
.../core/scan/filter/intf/RowImpl.java | 44 +
.../core/scan/filter/intf/RowIntf.java | 31 +
.../resolver/ConditionalFilterResolverImpl.java | 229 +++
.../filter/resolver/FilterResolverIntf.java | 100 ++
.../resolver/LogicalFilterResolverImpl.java | 115 ++
.../resolver/RowLevelFilterResolverImpl.java | 136 ++
.../RowLevelRangeFilterResolverImpl.java | 266 ++++
.../metadata/FilterResolverMetadata.java | 62 +
.../DimColumnResolvedFilterInfo.java | 143 ++
.../MeasureColumnResolvedFilterInfo.java | 60 +
.../visitor/CustomTypeDictionaryVisitor.java | 101 ++
.../visitor/DictionaryColumnVisitor.java | 67 +
.../visitor/FilterInfoTypeVisitorFactory.java | 45 +
.../visitor/NoDictionaryTypeVisitor.java | 64 +
.../visitor/ResolvedFilterInfoVisitorIntf.java | 42 +
.../core/scan/model/CarbonQueryPlan.java | 156 ++
.../carbondata/core/scan/model/QueryColumn.java | 68 +
.../core/scan/model/QueryDimension.java | 58 +
.../core/scan/model/QueryMeasure.java | 61 +
.../carbondata/core/scan/model/QueryModel.java | 365 +++++
.../processor/AbstractDataBlockIterator.java | 146 ++
.../core/scan/processor/BlockletIterator.java | 88 ++
.../core/scan/processor/BlocksChunkHolder.java | 125 ++
.../processor/impl/DataBlockIteratorImpl.java | 73 +
.../core/scan/result/AbstractScannedResult.java | 570 ++++++++
.../core/scan/result/BatchResult.java | 105 ++
.../result/impl/FilterQueryScannedResult.java | 182 +++
.../impl/NonFilterQueryScannedResult.java | 146 ++
.../AbstractDetailQueryResultIterator.java | 185 +++
.../scan/result/iterator/ChunkRowIterator.java | 79 +
.../iterator/DetailQueryResultIterator.java | 85 ++
.../scan/result/iterator/RawResultIterator.java | 169 +++
.../result/iterator/VectorChunkRowIterator.java | 93 ++
.../VectorDetailQueryResultIterator.java | 52 +
.../scan/result/vector/CarbonColumnVector.java | 47 +
.../scan/result/vector/CarbonColumnarBatch.java | 62 +
.../scan/result/vector/ColumnVectorInfo.java | 41 +
.../vector/MeasureDataVectorProcessor.java | 268 ++++
.../vector/impl/CarbonColumnVectorImpl.java | 154 ++
.../scan/scanner/AbstractBlockletScanner.java | 93 ++
.../core/scan/scanner/BlockletScanner.java | 42 +
.../core/scan/scanner/impl/FilterScanner.java | 200 +++
.../scan/scanner/impl/NonFilterScanner.java | 40 +
.../core/scan/wrappers/ByteArrayWrapper.java | 221 +++
.../core/service/CarbonCommonFactory.java | 54 +
.../service/impl/ColumnUniqueIdGenerator.java | 41 +
.../core/service/impl/DictionaryFactory.java | 119 ++
.../core/service/impl/PathFactory.java | 44 +
.../core/update/CarbonUpdateUtil.java | 10 +-
.../apache/carbondata/core/update/UpdateVO.java | 108 --
.../core/update/data/BlockMappingVO.java | 54 +
.../data/BlockletDeleteDeltaCacheLoader.java | 79 +
.../data/BlockletLevelDeleteDeltaDataCache.java | 46 +
.../update/data/DeleteDeltaCacheLoaderIntf.java | 27 +
.../core/update/data/RowCountDetailsVO.java | 68 +
.../carbondata/core/update/data/UpdateVO.java | 108 ++
.../core/updatestatus/SegmentStatusManager.java | 2 +-
.../SegmentUpdateStatusManager.java | 8 +-
.../core/util/CarbonMetadataUtil.java | 4 +-
.../apache/carbondata/core/util/CarbonUtil.java | 20 +-
.../core/util/DataFileFooterConverter.java | 4 +-
.../core/util/ValueCompressionUtil.java | 33 +-
.../writer/CarbonDeleteDeltaWriterImpl.java | 2 +-
.../core/writer/CarbonDictionaryWriterImpl.java | 6 +-
.../carbondata/core/writer/ThriftWriter.java | 2 +-
.../CarbonDictionarySortIndexWriterImpl.java | 6 +-
.../AtomicFileOperationsImpl.java | 6 +-
.../apache/carbondata/locks/HdfsFileLock.java | 4 +-
.../apache/carbondata/locks/LocalFileLock.java | 2 +-
.../scan/collector/ScannedResultCollector.java | 45 -
.../impl/AbstractScannedResultCollector.java | 159 --
.../impl/DictionaryBasedResultCollector.java | 172 ---
.../DictionaryBasedVectorResultCollector.java | 136 --
.../collector/impl/RawBasedResultCollector.java | 74 -
.../scan/complextypes/ArrayQueryType.java | 103 --
.../scan/complextypes/ComplexQueryType.java | 64 -
.../scan/complextypes/PrimitiveQueryType.java | 118 --
.../scan/complextypes/StructQueryType.java | 114 --
.../carbondata/scan/executor/QueryExecutor.java | 51 -
.../scan/executor/QueryExecutorFactory.java | 38 -
.../exception/QueryExecutionException.java | 94 --
.../executor/impl/AbstractQueryExecutor.java | 483 -------
.../scan/executor/impl/DetailQueryExecutor.java | 49 -
.../executor/impl/QueryExecutorProperties.java | 73 -
.../impl/VectorDetailQueryExecutor.java | 46 -
.../scan/executor/infos/AggregatorInfo.java | 105 --
.../scan/executor/infos/BlockExecutionInfo.java | 604 --------
.../scan/executor/infos/KeyStructureInfo.java | 100 --
.../scan/executor/util/QueryUtil.java | 882 ------------
.../scan/executor/util/RestructureUtil.java | 140 --
.../scan/expression/BinaryExpression.java | 43 -
.../scan/expression/ColumnExpression.java | 114 --
.../carbondata/scan/expression/Expression.java | 49 -
.../scan/expression/ExpressionResult.java | 545 -------
.../scan/expression/LeafExpression.java | 24 -
.../scan/expression/LiteralExpression.java | 69 -
.../scan/expression/UnknownExpression.java | 28 -
.../BinaryConditionalExpression.java | 37 -
.../conditional/ConditionalExpression.java | 37 -
.../conditional/EqualToExpression.java | 109 --
.../GreaterThanEqualToExpression.java | 91 --
.../conditional/GreaterThanExpression.java | 94 --
.../expression/conditional/InExpression.java | 99 --
.../conditional/LessThanEqualToExpression.java | 93 --
.../conditional/LessThanExpression.java | 96 --
.../expression/conditional/ListExpression.java | 62 -
.../conditional/NotEqualsExpression.java | 105 --
.../expression/conditional/NotInExpression.java | 98 --
.../exception/FilterIllegalMemberException.java | 94 --
.../exception/FilterUnsupportedException.java | 88 --
.../scan/expression/logical/AndExpression.java | 63 -
.../logical/BinaryLogicalExpression.java | 109 --
.../expression/logical/FalseExpression.java | 70 -
.../scan/expression/logical/OrExpression.java | 62 -
.../scan/filter/DimColumnFilterInfo.java | 80 --
.../scan/filter/FilterExpressionProcessor.java | 338 -----
.../carbondata/scan/filter/FilterProcessor.java | 57 -
.../carbondata/scan/filter/FilterUtil.java | 1353 ------------------
.../scan/filter/GenericQueryType.java | 49 -
.../filter/executer/AndFilterExecuterImpl.java | 63 -
.../executer/DimColumnExecuterFilterInfo.java | 32 -
.../ExcludeColGroupFilterExecuterImpl.java | 135 --
.../executer/ExcludeFilterExecuterImpl.java | 153 --
.../scan/filter/executer/FilterExecuter.java | 47 -
.../IncludeColGroupFilterExecuterImpl.java | 207 ---
.../executer/IncludeFilterExecuterImpl.java | 178 ---
.../filter/executer/OrFilterExecuterImpl.java | 53 -
.../executer/RestructureFilterExecuterImpl.java | 54 -
.../executer/RowLevelFilterExecuterImpl.java | 368 -----
.../RowLevelRangeGrtThanFiterExecuterImpl.java | 212 ---
...elRangeGrtrThanEquaToFilterExecuterImpl.java | 199 ---
...velRangeLessThanEqualFilterExecuterImpl.java | 248 ----
.../RowLevelRangeLessThanFiterExecuterImpl.java | 255 ----
.../RowLevelRangeTypeExecuterFacory.java | 92 --
.../scan/filter/intf/ExpressionType.java | 45 -
.../scan/filter/intf/FilterExecuterType.java | 28 -
.../carbondata/scan/filter/intf/RowImpl.java | 44 -
.../carbondata/scan/filter/intf/RowIntf.java | 31 -
.../resolver/ConditionalFilterResolverImpl.java | 229 ---
.../filter/resolver/FilterResolverIntf.java | 100 --
.../resolver/LogicalFilterResolverImpl.java | 115 --
.../resolver/RowLevelFilterResolverImpl.java | 136 --
.../RowLevelRangeFilterResolverImpl.java | 266 ----
.../metadata/FilterResolverMetadata.java | 62 -
.../DimColumnResolvedFilterInfo.java | 143 --
.../MeasureColumnResolvedFilterInfo.java | 60 -
.../visitor/CustomTypeDictionaryVisitor.java | 101 --
.../visitor/DictionaryColumnVisitor.java | 67 -
.../visitor/FilterInfoTypeVisitorFactory.java | 45 -
.../visitor/NoDictionaryTypeVisitor.java | 64 -
.../visitor/ResolvedFilterInfoVisitorIntf.java | 42 -
.../carbondata/scan/model/CarbonQueryPlan.java | 156 --
.../carbondata/scan/model/QueryColumn.java | 68 -
.../carbondata/scan/model/QueryDimension.java | 58 -
.../carbondata/scan/model/QueryMeasure.java | 61 -
.../carbondata/scan/model/QueryModel.java | 365 -----
.../processor/AbstractDataBlockIterator.java | 146 --
.../scan/processor/BlockletIterator.java | 88 --
.../scan/processor/BlocksChunkHolder.java | 125 --
.../processor/impl/DataBlockIteratorImpl.java | 73 -
.../scan/result/AbstractScannedResult.java | 570 --------
.../carbondata/scan/result/BatchResult.java | 105 --
.../result/impl/FilterQueryScannedResult.java | 182 ---
.../impl/NonFilterQueryScannedResult.java | 146 --
.../AbstractDetailQueryResultIterator.java | 185 ---
.../scan/result/iterator/ChunkRowIterator.java | 79 -
.../iterator/DetailQueryResultIterator.java | 85 --
.../scan/result/iterator/RawResultIterator.java | 169 ---
.../result/iterator/VectorChunkRowIterator.java | 93 --
.../VectorDetailQueryResultIterator.java | 52 -
.../scan/result/vector/CarbonColumnVector.java | 47 -
.../scan/result/vector/CarbonColumnarBatch.java | 62 -
.../scan/result/vector/ColumnVectorInfo.java | 41 -
.../vector/MeasureDataVectorProcessor.java | 268 ----
.../vector/impl/CarbonColumnVectorImpl.java | 154 --
.../scan/scanner/AbstractBlockletScanner.java | 93 --
.../scan/scanner/BlockletScanner.java | 42 -
.../scan/scanner/impl/FilterScanner.java | 200 ---
.../scan/scanner/impl/NonFilterScanner.java | 40 -
.../scan/wrappers/ByteArrayWrapper.java | 221 ---
.../core/AbsoluteTableIdentifierTest.java | 105 ++
.../core/CarbonTableIdentifierTest.java | 98 ++
.../carbondata/core/ColumnIdentifierTest.java | 100 ++
.../dictionary/AbstractDictionaryCacheTest.java | 4 +-
.../carbon/AbsoluteTableIdentifierTest.java | 105 --
.../core/carbon/CarbonTableIdentifierTest.java | 98 --
.../core/carbon/ColumnIdentifierTest.java | 100 --
.../impl/CompressedDataMeasureWrapperTest.java | 47 -
.../store/impl/DFSFileHolderImplUnitTest.java | 153 --
.../store/impl/FileFactoryImplUnitTest.java | 153 --
.../store/impl/FileHolderImplUnitTest.java | 157 --
.../DriverQueryStatisticsRecorderImplTest.java | 124 --
.../QueryStasticsRecorderImplTest.java | 114 --
.../CompressedDataMeasureWrapperTest.java | 47 +
.../datastorage/DFSFileHolderImplUnitTest.java | 153 ++
.../datastorage/FileFactoryImplUnitTest.java | 153 ++
.../datastorage/FileHolderImplUnitTest.java | 157 ++
.../ColumnarKeyStoreDataHolderUnitTest.java | 93 ++
.../filesystem/AlluxioCarbonFileTest.java | 292 ++++
.../filesystem/HDFSCarbonFileTest.java | 400 ++++++
.../filesystem/LocalCarbonFileTest.java | 471 ++++++
.../filesystem/ViewFsCarbonFileTest.java | 311 ++++
.../ColumnarKeyStoreDataHolderUnitTest.java | 93 --
.../store/filesystem/AlluxioCarbonFileTest.java | 295 ----
.../store/filesystem/HDFSCarbonFileTest.java | 402 ------
.../store/filesystem/LocalCarbonFileTest.java | 472 ------
.../store/filesystem/ViewFsCarbonFileTest.java | 314 ----
.../impl/ColumnGroupDimensionDataChunkTest.java | 4 +-
.../impl/FixedLengthDimensionDataChunkTest.java | 2 +-
...mpressedMeasureChunkFileBasedReaderTest.java | 30 +-
.../reader/CarbonDictionaryReaderImplTest.java | 4 +-
...CarbonDictionarySortIndexReaderImplTest.java | 4 +-
.../DictionaryBasedResultCollectorTest.java | 154 ++
.../impl/RawBasedResultCollectorTest.java | 132 ++
.../scan/complextypes/ArrayQueryTypeTest.java | 57 +
.../complextypes/PrimitiveQueryTypeTest.java | 127 ++
.../scan/complextypes/StructQueryTypeTest.java | 69 +
.../core/scan/executor/util/QueryUtilTest.java | 241 ++++
.../scan/executor/util/RestructureUtilTest.java | 115 ++
.../scan/expression/ColumnExpressionTest.java | 71 +
.../scan/expression/ExpressionResultTest.java | 498 +++++++
.../scan/expression/LiteralExpressionTest.java | 59 +
.../conditional/EqualToExpressionUnitTest.java | 315 ++++
.../GreaterThanEqualToExpressionUnitTest.java | 293 ++++
.../GreaterThanExpressionUnitTest.java | 364 +++++
.../conditional/InExpressionUnitTest.java | 276 ++++
.../LessThanEqualToExpressionUnitTest.java | 368 +++++
.../conditional/LessThanExpressionUnitTest.java | 364 +++++
.../conditional/ListExpressionUnitTest.java | 64 +
.../NotEqualsExpressionUnitTest.java | 406 ++++++
.../conditional/NotInExpressionUnitTest.java | 275 ++++
.../expression/logical/AndExpressionTest.java | 81 ++
.../expression/logical/FalseExpressionTest.java | 48 +
.../expression/logical/OrExpressionTest.java | 79 +
.../core/scan/filter/FilterUtilTest.java | 392 +++++
.../core/scan/result/BatchResultTest.java | 95 ++
.../impl/FilterQueryScannedResultTest.java | 210 +++
.../impl/NonFilterQueryScannedResultTest.java | 209 +++
.../scan/wrappers/ByteArrayWrapperTest.java | 107 ++
.../DriverQueryStatisticsRecorderImplTest.java | 124 ++
.../stats/QueryStasticsRecorderImplTest.java | 114 ++
.../core/util/CarbonMetadataUtilTest.java | 2 +-
.../carbondata/core/util/CarbonUtilTest.java | 10 +-
.../core/util/DataFileFooterConverterTest.java | 6 +-
.../core/util/ValueCompressionUtilTest.java | 29 +-
.../writer/CarbonDictionaryWriterImplTest.java | 4 +-
.../core/writer/CarbonFooterWriterTest.java | 6 +-
.../DictionaryBasedResultCollectorTest.java | 154 --
.../impl/RawBasedResultCollectorTest.java | 132 --
.../scan/complextypes/ArrayQueryTypeTest.java | 57 -
.../complextypes/PrimitiveQueryTypeTest.java | 132 --
.../scan/complextypes/StructQueryTypeTest.java | 69 -
.../scan/executor/util/QueryUtilTest.java | 241 ----
.../scan/executor/util/RestructureUtilTest.java | 115 --
.../scan/expression/ColumnExpressionTest.java | 71 -
.../scan/expression/ExpressionResultTest.java | 498 -------
.../scan/expression/LiteralExpressionTest.java | 59 -
.../conditional/EqualToExpressionUnitTest.java | 315 ----
.../GreaterThanEqualToExpressionUnitTest.java | 293 ----
.../GreaterThanExpressionUnitTest.java | 364 -----
.../conditional/InExpressionUnitTest.java | 276 ----
.../LessThanEqualToExpressionUnitTest.java | 368 -----
.../conditional/LessThanExpressionUnitTest.java | 364 -----
.../conditional/ListExpressionUnitTest.java | 64 -
.../NotEqualsExpressionUnitTest.java | 406 ------
.../conditional/NotInExpressionUnitTest.java | 275 ----
.../expression/logical/AndExpressionTest.java | 81 --
.../expression/logical/FalseExpressionTest.java | 48 -
.../expression/logical/OrExpressionTest.java | 79 -
.../carbondata/scan/filter/FilterUtilTest.java | 392 -----
.../carbondata/scan/result/BatchResultTest.java | 95 --
.../impl/FilterQueryScannedResultTest.java | 210 ---
.../impl/NonFilterQueryScannedResultTest.java | 209 ---
.../scan/wrappers/ByteArrayWrapperTest.java | 107 --
.../scanner/impl/FilterScannerTest.java | 162 ---
dev/findbugs-exclude.xml | 4 +-
.../carbondata/examples/AlluxioExample.scala | 2 +-
.../examples/util/AllDictionaryUtil.scala | 2 +-
.../carbondata/hadoop/CarbonInputFormat.java | 16 +-
.../carbondata/hadoop/CarbonInputSplit.java | 2 +-
.../carbondata/hadoop/CarbonRecordReader.java | 10 +-
.../hadoop/api/CarbonTableInputFormat.java | 4 +-
.../carbondata/hadoop/internal/index/Index.java | 2 +-
.../internal/index/impl/InMemoryBTreeIndex.java | 6 +-
.../hadoop/internal/segment/Segment.java | 2 +-
.../internal/segment/impl/IndexedSegment.java | 2 +-
.../hadoop/util/BlockLevelTraverser.java | 2 +-
.../hadoop/util/CarbonInputFormatUtil.java | 14 +-
.../carbondata/hadoop/util/SchemaReader.java | 2 +-
.../hadoop/ft/CarbonInputFormat_FT.java | 8 +-
.../hadoop/ft/CarbonInputMapperTest.java | 8 +-
.../test/util/ObjectSerializationUtilTest.java | 8 +-
.../hadoop/test/util/StoreCreator.java | 2 +-
.../dataload/TestLoadDataGeneral.scala | 2 +-
.../blockprune/BlockPruneQueryTestCase.scala | 2 +-
.../SubqueryWithFilterAndSortTestCase.scala | 2 +-
.../ValueCompressionDataTypeTestCase.scala | 2 +-
.../carbondata/spark/load/CarbonLoaderUtil.java | 8 +-
.../spark/load/DeleteLoadFolders.java | 6 +-
.../spark/merger/CarbonCompactionExecutor.java | 16 +-
.../spark/merger/CarbonCompactionUtil.java | 2 +-
.../spark/merger/CarbonDataMergerUtil.java | 6 +-
.../spark/merger/RowResultMerger.java | 4 +-
.../spark/merger/TupleConversionAdapter.java | 2 +-
.../api/impl/QueryPartitionHelper.java | 2 +-
.../carbondata/spark/util/CarbonQueryUtil.java | 2 +-
.../apache/carbondata/spark/CarbonFilters.scala | 6 +-
.../spark/DictionaryDetailHelper.scala | 6 +-
.../spark/rdd/CarbonGlobalDictionaryRDD.scala | 5 +-
.../carbondata/spark/rdd/CarbonMergerRDD.scala | 4 +-
.../carbondata/spark/rdd/CarbonScanRDD.scala | 4 +-
.../spark/tasks/DictionaryWriterTask.scala | 2 +-
.../spark/tasks/SortIndexWriterTask.scala | 2 +-
.../carbondata/spark/util/CarbonScalaUtil.scala | 2 +-
.../spark/util/GlobalDictionaryUtil.scala | 4 +-
.../execution/command/carbonTableSchema.scala | 2 +-
.../scala/org/apache/spark/util/FileUtils.scala | 4 +-
.../apache/spark/sql/CarbonBoundReference.scala | 2 +-
.../sql/CarbonDatasourceHadoopRelation.scala | 2 +-
.../spark/sql/CarbonDatasourceRelation.scala | 2 +-
.../scala/org/apache/spark/sql/CarbonScan.scala | 2 +-
.../spark/sql/SparkUnknownExpression.scala | 8 +-
.../sql/execution/command/IUDCommands.scala | 6 +-
.../execution/command/carbonTableSchema.scala | 2 +-
.../apache/spark/sql/hive/CarbonMetastore.scala | 4 +-
.../CompactionSystemLockFeatureTest.scala | 2 +-
...GlobalDictionaryUtilConcurrentTestCase.scala | 4 +-
.../readsupport/SparkRowReadSupportImpl.java | 4 +-
.../vectorreader/ColumnarVectorWrapper.java | 2 +-
.../VectorizedCarbonRecordReader.java | 22 +-
.../apache/carbondata/spark/CarbonFilters.scala | 8 +-
.../carbondata/spark/CarbonSparkFactory.scala | 2 +-
.../spark/DictionaryDetailHelper.scala | 6 +-
.../spark/rdd/CarbonDataRDDFactory.scala | 2 +-
.../carbondata/spark/util/CarbonSparkUtil.scala | 2 +-
.../carbondata/spark/util/QueryPlanUtil.scala | 2 +-
.../spark/sql/CarbonDataFrameWriter.scala | 2 +-
.../sql/CarbonDatasourceHadoopRelation.scala | 6 +-
.../spark/sql/CarbonDictionaryDecoder.scala | 6 +-
.../scala/org/apache/spark/sql/CarbonScan.scala | 2 +-
.../spark/sql/SparkUnknownExpression.scala | 10 +-
.../org/apache/spark/sql/TableCreator.scala | 2 +-
.../execution/command/carbonTableSchema.scala | 9 +-
.../apache/spark/sql/hive/CarbonMetastore.scala | 34 +-
.../sql/optimizer/CarbonLateDecodeRule.scala | 2 +-
.../processing/csvload/DataGraphExecuter.java | 8 +-
.../processing/csvload/GraphExecutionUtil.java | 8 +-
.../csvreaderstep/UnivocityCsvParser.java | 4 +-
...ConverterProcessorWithBucketingStepImpl.java | 4 +-
.../newflow/steps/partition/Partitioner.java | 26 +
.../partition/impl/HashPartitionerImpl.java | 105 ++
.../sortdata/CompressedTempSortFileWriter.java | 2 +-
.../store/CarbonFactDataHandlerColumnar.java | 13 +-
.../processing/store/StoreFactory.java | 35 +
.../store/colgroup/ColGroupBlockStorage.java | 2 +-
.../store/writer/AbstractFactDataWriter.java | 8 +-
.../store/writer/CarbonFactDataWriter.java | 4 +-
.../processing/store/writer/NodeHolder.java | 2 +-
.../writer/v1/CarbonFactDataWriterImplV1.java | 4 +-
.../csvbased/BadRecordsLogger.java | 4 +-
.../util/CarbonDataProcessorUtil.java | 8 +-
.../carbondata/test/util/StoreCreator.java | 2 +-
589 files changed, 31295 insertions(+), 31447 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/common/ext/ColumnUniqueIdGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/common/ext/ColumnUniqueIdGenerator.java b/core/src/main/java/org/apache/carbondata/common/ext/ColumnUniqueIdGenerator.java
deleted file mode 100644
index 1f02e9a..0000000
--- a/core/src/main/java/org/apache/carbondata/common/ext/ColumnUniqueIdGenerator.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.common.ext;
-
-import java.util.UUID;
-
-import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
-import org.apache.carbondata.core.service.ColumnUniqueIdService;
-
-/**
- * It returns unique id given column
- */
-public class ColumnUniqueIdGenerator implements ColumnUniqueIdService {
-
- private static ColumnUniqueIdService columnUniqueIdService = new ColumnUniqueIdGenerator();
-
- @Override public String generateUniqueId(String databaseName, ColumnSchema columnSchema) {
- return UUID.randomUUID().toString();
- }
-
- public static ColumnUniqueIdService getInstance() {
- return columnUniqueIdService;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/common/ext/DictionaryFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/common/ext/DictionaryFactory.java b/core/src/main/java/org/apache/carbondata/common/ext/DictionaryFactory.java
deleted file mode 100644
index a3b7639..0000000
--- a/core/src/main/java/org/apache/carbondata/common/ext/DictionaryFactory.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.common.ext;
-
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.ColumnIdentifier;
-import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader;
-import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReaderImpl;
-import org.apache.carbondata.core.reader.CarbonDictionaryReader;
-import org.apache.carbondata.core.reader.CarbonDictionaryReaderImpl;
-import org.apache.carbondata.core.reader.sortindex.CarbonDictionarySortIndexReader;
-import org.apache.carbondata.core.reader.sortindex.CarbonDictionarySortIndexReaderImpl;
-import org.apache.carbondata.core.service.DictionaryService;
-import org.apache.carbondata.core.writer.CarbonDictionaryWriter;
-import org.apache.carbondata.core.writer.CarbonDictionaryWriterImpl;
-import org.apache.carbondata.core.writer.sortindex.CarbonDictionarySortIndexWriter;
-import org.apache.carbondata.core.writer.sortindex.CarbonDictionarySortIndexWriterImpl;
-
-/**
- * service to get dictionary reader and writer
- */
-public class DictionaryFactory implements DictionaryService {
-
- private static DictionaryService dictService = new DictionaryFactory();
-
- /**
- * get dictionary writer
- *
- * @param carbonTableIdentifier
- * @param columnIdentifier
- * @param carbonStorePath
- * @return
- */
- @Override public CarbonDictionaryWriter getDictionaryWriter(
- CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier,
- String carbonStorePath) {
- return new CarbonDictionaryWriterImpl(carbonStorePath, carbonTableIdentifier, columnIdentifier);
- }
-
- /**
- * get dictionary sort index writer
- *
- * @param carbonTableIdentifier
- * @param columnIdentifier
- * @param carbonStorePath
- * @return
- */
- @Override public CarbonDictionarySortIndexWriter getDictionarySortIndexWriter(
- CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier,
- String carbonStorePath) {
- return new CarbonDictionarySortIndexWriterImpl(carbonTableIdentifier, columnIdentifier,
- carbonStorePath);
- }
-
- /**
- * get dictionary metadata reader
- *
- * @param carbonTableIdentifier
- * @param columnIdentifier
- * @param carbonStorePath
- * @return
- */
- @Override public CarbonDictionaryMetadataReader getDictionaryMetadataReader(
- CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier,
- String carbonStorePath) {
- return new CarbonDictionaryMetadataReaderImpl(carbonStorePath, carbonTableIdentifier,
- columnIdentifier);
- }
-
- /**
- * get dictionary reader
- *
- * @param carbonTableIdentifier
- * @param columnIdentifier
- * @param carbonStorePath
- * @return
- */
- @Override public CarbonDictionaryReader getDictionaryReader(
- CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier,
- String carbonStorePath) {
- return new CarbonDictionaryReaderImpl(carbonStorePath, carbonTableIdentifier, columnIdentifier);
- }
-
- /**
- * get dictionary sort index reader
- *
- * @param carbonTableIdentifier
- * @param columnIdentifier
- * @param carbonStorePath
- * @return
- */
- @Override public CarbonDictionarySortIndexReader getDictionarySortIndexReader(
- CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier,
- String carbonStorePath) {
- return new CarbonDictionarySortIndexReaderImpl(carbonTableIdentifier, columnIdentifier,
- carbonStorePath);
- }
-
- public static DictionaryService getInstance() {
- return dictService;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/common/ext/PathFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/common/ext/PathFactory.java b/core/src/main/java/org/apache/carbondata/common/ext/PathFactory.java
deleted file mode 100644
index b175334..0000000
--- a/core/src/main/java/org/apache/carbondata/common/ext/PathFactory.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.common.ext;
-
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.path.CarbonStorePath;
-import org.apache.carbondata.core.path.CarbonTablePath;
-import org.apache.carbondata.core.service.PathService;
-
-/**
- * Create helper to get path details
- */
-public class PathFactory implements PathService {
-
- private static PathService pathService = new PathFactory();
-
- /**
- * @param storeLocation
- * @param tableIdentifier
- * @return store path related to tables
- */
- @Override public CarbonTablePath getCarbonTablePath(
- String storeLocation, CarbonTableIdentifier tableIdentifier) {
- return CarbonStorePath.getCarbonTablePath(storeLocation, tableIdentifier);
- }
-
- public static PathService getInstance() {
- return pathService;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/common/factory/CarbonCommonFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/common/factory/CarbonCommonFactory.java b/core/src/main/java/org/apache/carbondata/common/factory/CarbonCommonFactory.java
deleted file mode 100644
index 09b4465..0000000
--- a/core/src/main/java/org/apache/carbondata/common/factory/CarbonCommonFactory.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.common.factory;
-
-import org.apache.carbondata.common.ext.ColumnUniqueIdGenerator;
-import org.apache.carbondata.common.ext.DictionaryFactory;
-import org.apache.carbondata.common.ext.PathFactory;
-import org.apache.carbondata.core.service.ColumnUniqueIdService;
-import org.apache.carbondata.core.service.DictionaryService;
-import org.apache.carbondata.core.service.PathService;
-
-/**
- * Interface to get services
- */
-public class CarbonCommonFactory {
-
- /**
- * @return dictionary service
- */
- public static DictionaryService getDictionaryService() {
- return DictionaryFactory.getInstance();
- }
-
- /**
- * @return path service
- */
- public static PathService getPathService() {
- return PathFactory.getInstance();
- }
-
- /**
- * @return unique id generator
- */
- public static ColumnUniqueIdService getColumnUniqueIdGenerator() {
- return ColumnUniqueIdGenerator.getInstance();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/common/iudprocessor/cache/BlockletLevelDeleteDeltaDataCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/common/iudprocessor/cache/BlockletLevelDeleteDeltaDataCache.java b/core/src/main/java/org/apache/carbondata/common/iudprocessor/cache/BlockletLevelDeleteDeltaDataCache.java
deleted file mode 100644
index acc00f0..0000000
--- a/core/src/main/java/org/apache/carbondata/common/iudprocessor/cache/BlockletLevelDeleteDeltaDataCache.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.common.iudprocessor.cache;
-
-import org.roaringbitmap.RoaringBitmap;
-
-/**
- * This class maintains delete delta data cache of each blocklet along with the block timestamp
- */
-public class BlockletLevelDeleteDeltaDataCache {
- private RoaringBitmap deleteDelataDataCache;
- private String timeStamp;
-
- public BlockletLevelDeleteDeltaDataCache(int[] deleteDeltaFileData, String timeStamp) {
- deleteDelataDataCache = RoaringBitmap.bitmapOf(deleteDeltaFileData);
- this.timeStamp=timeStamp;
- }
-
- public boolean contains(int key) {
- return deleteDelataDataCache.contains(key);
- }
-
- public int getSize() {
- return deleteDelataDataCache.getCardinality();
- }
-
- public String getCacheTimeStamp() {
- return timeStamp;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/BlockMappingVO.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/BlockMappingVO.java b/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/BlockMappingVO.java
deleted file mode 100644
index ab43b6b..0000000
--- a/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/BlockMappingVO.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.common.iudprocessor.iuddata;
-
-import java.util.Map;
-
-/**
- * VO class to store the details of segment and block count , block and its row count.
- */
-public class BlockMappingVO {
-
- private Map<String, Long> blockRowCountMapping ;
-
- private Map<String, Long> segmentNumberOfBlockMapping ;
-
- private Map<String, RowCountDetailsVO> completeBlockRowDetailVO;
-
- public void setCompleteBlockRowDetailVO(Map<String, RowCountDetailsVO> completeBlockRowDetailVO) {
- this.completeBlockRowDetailVO = completeBlockRowDetailVO;
- }
-
- public Map<String, RowCountDetailsVO> getCompleteBlockRowDetailVO() {
- return completeBlockRowDetailVO;
- }
-
- public Map<String, Long> getBlockRowCountMapping() {
- return blockRowCountMapping;
- }
-
- public Map<String, Long> getSegmentNumberOfBlockMapping() {
- return segmentNumberOfBlockMapping;
- }
-
- public BlockMappingVO(Map<String, Long> blockRowCountMapping,
- Map<String, Long> segmentNumberOfBlockMapping) {
- this.blockRowCountMapping = blockRowCountMapping;
- this.segmentNumberOfBlockMapping = segmentNumberOfBlockMapping;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/BlockletDeleteDeltaCacheLoader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/BlockletDeleteDeltaCacheLoader.java b/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/BlockletDeleteDeltaCacheLoader.java
deleted file mode 100644
index 1af983b..0000000
--- a/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/BlockletDeleteDeltaCacheLoader.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.common.iudprocessor.iuddata;
-
-import org.apache.carbondata.common.iudprocessor.cache.BlockletLevelDeleteDeltaDataCache;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.datastore.DataRefNode;
-import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager;
-
-/**
- * This class is responsible for loading delete delta file cache based on
- * blocklet id of a particular block
- */
-public class BlockletDeleteDeltaCacheLoader implements DeleteDeltaCacheLoaderIntf {
- private String blockletID;
- private DataRefNode blockletNode;
- private AbsoluteTableIdentifier absoluteIdentifier;
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(BlockletDeleteDeltaCacheLoader.class.getName());
-
- public BlockletDeleteDeltaCacheLoader(String blockletID,
- DataRefNode blockletNode, AbsoluteTableIdentifier absoluteIdentifier) {
- this.blockletID = blockletID;
- this.blockletNode = blockletNode;
- this.absoluteIdentifier= absoluteIdentifier;
- }
-
- /**
- * This method will load the delete delta cache based on blocklet id of particular block with
- * the help of SegmentUpdateStatusManager.
- */
- public void loadDeleteDeltaFileDataToCache() {
- SegmentUpdateStatusManager segmentUpdateStatusManager =
- new SegmentUpdateStatusManager(absoluteIdentifier);
- int[] deleteDeltaFileData = null;
- BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache = null;
- if (null == blockletNode.getDeleteDeltaDataCache()) {
- try {
- deleteDeltaFileData = segmentUpdateStatusManager.getDeleteDeltaDataFromAllFiles(blockletID);
- deleteDeltaDataCache = new BlockletLevelDeleteDeltaDataCache(deleteDeltaFileData,
- segmentUpdateStatusManager.getTimestampForRefreshCache(blockletID, null));
- } catch (Exception e) {
- LOGGER.debug("Unable to retrieve delete delta files");
- }
- } else {
- deleteDeltaDataCache = blockletNode.getDeleteDeltaDataCache();
- // if already cache is present then validate the cache using timestamp
- String cacheTimeStamp = segmentUpdateStatusManager
- .getTimestampForRefreshCache(blockletID, deleteDeltaDataCache.getCacheTimeStamp());
- if (null != cacheTimeStamp) {
- try {
- deleteDeltaFileData =
- segmentUpdateStatusManager.getDeleteDeltaDataFromAllFiles(blockletID);
- deleteDeltaDataCache = new BlockletLevelDeleteDeltaDataCache(deleteDeltaFileData,
- segmentUpdateStatusManager.getTimestampForRefreshCache(blockletID, cacheTimeStamp));
- } catch (Exception e) {
- LOGGER.debug("Unable to retrieve delete delta files");
- }
- }
- }
- blockletNode.setDeleteDeltaDataCache(deleteDeltaDataCache);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/DeleteDeltaCacheLoaderIntf.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/DeleteDeltaCacheLoaderIntf.java b/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/DeleteDeltaCacheLoaderIntf.java
deleted file mode 100644
index 2455910..0000000
--- a/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/DeleteDeltaCacheLoaderIntf.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.common.iudprocessor.iuddata;
-
-/**
- * This interface holds all methods required to load delete delta file data to cache
- */
-public interface DeleteDeltaCacheLoaderIntf {
-
- void loadDeleteDeltaFileDataToCache();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/RowCountDetailsVO.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/RowCountDetailsVO.java b/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/RowCountDetailsVO.java
deleted file mode 100644
index 7b5cc31..0000000
--- a/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/RowCountDetailsVO.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.common.iudprocessor.iuddata;
-
-import java.io.Serializable;
-
-/**
- * VO class Details for block.
- */
-public class RowCountDetailsVO implements Serializable {
-
- private static final long serialVersionUID = 1206104914918491749L;
-
- private long totalNumberOfRows;
-
- private long deletedRowsInBlock;
-
- public RowCountDetailsVO(long totalNumberOfRows, long deletedRowsInBlock) {
- this.totalNumberOfRows = totalNumberOfRows;
- this.deletedRowsInBlock = deletedRowsInBlock;
- }
-
- public long getTotalNumberOfRows() {
- return totalNumberOfRows;
- }
-
- public long getDeletedRowsInBlock() {
- return deletedRowsInBlock;
- }
-
- @Override public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null || getClass() != obj.getClass()) {
- return false;
- }
-
- RowCountDetailsVO that = (RowCountDetailsVO) obj;
-
- if (totalNumberOfRows != that.totalNumberOfRows) {
- return false;
- }
- return deletedRowsInBlock == that.deletedRowsInBlock;
-
- }
-
- @Override public int hashCode() {
- int result = (int) (totalNumberOfRows ^ (totalNumberOfRows >>> 32));
- result = 31 * result + (int) (deletedRowsInBlock ^ (deletedRowsInBlock >>> 32));
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/AbsoluteTableIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/AbsoluteTableIdentifier.java b/core/src/main/java/org/apache/carbondata/core/AbsoluteTableIdentifier.java
index c07b8bf..e5823b9 100644
--- a/core/src/main/java/org/apache/carbondata/core/AbsoluteTableIdentifier.java
+++ b/core/src/main/java/org/apache/carbondata/core/AbsoluteTableIdentifier.java
@@ -22,7 +22,7 @@ import java.io.File;
import java.io.Serializable;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.util.CarbonUtil;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
index 9143eb0..b1bf459 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
@@ -22,14 +22,14 @@ package org.apache.carbondata.core.cache.dictionary;
import java.io.IOException;
import java.util.List;
-import org.apache.carbondata.common.factory.CarbonCommonFactory;
+import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.core.cache.Cache;
import org.apache.carbondata.core.cache.CacheType;
import org.apache.carbondata.core.cache.CarbonLRUCache;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk;
import org.apache.carbondata.core.reader.CarbonDictionaryMetadataReader;
import org.apache.carbondata.core.service.DictionaryService;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
index 6a6a86d..7d57676 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImpl.java
@@ -24,7 +24,7 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import org.apache.carbondata.common.factory.CarbonCommonFactory;
+import org.apache.carbondata.core.service.CarbonCommonFactory;
import org.apache.carbondata.core.CarbonTableIdentifier;
import org.apache.carbondata.core.ColumnIdentifier;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java
index 96ca2e1..4da8795 100644
--- a/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/compression/BigDecimalCompressor.java
@@ -18,7 +18,7 @@
*/
package org.apache.carbondata.core.compression;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder;
+import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder;
import org.apache.carbondata.core.util.BigDecimalCompressionFinder;
import org.apache.carbondata.core.util.CompressionFinder;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
index c27887d..e509c5c 100644
--- a/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/compression/BigIntCompressor.java
@@ -18,7 +18,7 @@
*/
package org.apache.carbondata.core.compression;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder;
+import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
index 383a515..a716921 100644
--- a/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/compression/DoubleCompressor.java
@@ -20,7 +20,7 @@ package org.apache.carbondata.core.compression;
import java.math.BigDecimal;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder;
+import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java b/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
index 31f86c0..a2d7b51 100644
--- a/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/compression/ValueCompressor.java
@@ -18,7 +18,7 @@
*/
package org.apache.carbondata.core.compression;
-import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder;
+import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder;
import org.apache.carbondata.core.util.CompressionFinder;
import org.apache.carbondata.core.util.ValueCompressionUtil.COMPRESSION_TYPE;
import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/FileHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/FileHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/FileHolder.java
new file mode 100644
index 0000000..96aa069
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/FileHolder.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage;
+
+import java.io.IOException;
+
+public interface FileHolder {
+ /**
+ * This method will be used to read the byte array from file based on offset
+ * and length(number of bytes) need to read
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @param length number of bytes to be read
+ * @return read byte array
+ */
+ byte[] readByteArray(String filePath, long offset, int length) throws IOException;
+
+ /**
+ * This method will be used to read the byte array from file based on length(number of bytes)
+ *
+ * @param filePath fully qualified file path
+ * @param length number of bytes to be read
+ * @return read byte array
+ */
+ byte[] readByteArray(String filePath, int length) throws IOException;
+
+ /**
+ * This method will be used to read int from file from postion(offset), here
+ * length will be always 4 bacause int byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read int
+ */
+ int readInt(String filePath, long offset) throws IOException;
+
+ /**
+ * This method will be used to read long from file from postion(offset), here
+ * length will be always 8 bacause int byte size is 8
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read long
+ */
+ long readLong(String filePath, long offset) throws IOException;
+
+ /**
+ * This method will be used to read int from file from postion(offset), here
+ * length will be always 4 bacause int byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @return read int
+ */
+ int readInt(String filePath) throws IOException;
+
+ /**
+ * This method will be used to read long value from file from postion(offset), here
+ * length will be always 8 because long byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read long
+ */
+ long readDouble(String filePath, long offset) throws IOException;
+
+ /**
+ * This method will be used to close all the streams currently present in the cache
+ */
+ void finish() throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/MeasureDataWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/MeasureDataWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastorage/MeasureDataWrapper.java
new file mode 100644
index 0000000..6fc6c1d
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/MeasureDataWrapper.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage;
+
+import org.apache.carbondata.core.datastorage.dataholder.CarbonReadDataHolder;
+
+/**
+ * MeasureDataWrapper, interface.
+ */
+public interface MeasureDataWrapper {
+ CarbonReadDataHolder[] getValues();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/NodeMeasureDataStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/NodeMeasureDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/NodeMeasureDataStore.java
new file mode 100644
index 0000000..fede06c
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/NodeMeasureDataStore.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage;
+
+import org.apache.carbondata.core.datastorage.dataholder.CarbonWriteDataHolder;
+
+public interface NodeMeasureDataStore {
+ /**
+ * This method will be used to get the writable key array.
+ * writable measure data array will hold below information:
+ * <size of measure data array><measure data array>
+ * total length will be 4 bytes for size + measure data array length
+ *
+ * @return writable array (compressed or normal)
+ */
+ byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] dataHolderArray);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForInt.java
new file mode 100644
index 0000000..0a2a8bd
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForInt.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.columnar;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.ByteUtil;
+
+public class BlockIndexerStorageForInt implements IndexStorage<int[]> {
+ private boolean alreadySorted;
+
+ private int[] dataAfterComp;
+
+ private int[] indexMap;
+
+ private byte[][] keyBlock;
+
+ private int[] dataIndexMap;
+
+ private int totalSize;
+
+ public BlockIndexerStorageForInt(byte[][] keyBlock, boolean compressData, boolean isNoDictionary,
+ boolean isSortRequired) {
+ ColumnWithIntIndex[] columnWithIndexs = createColumnWithIndexArray(keyBlock, isNoDictionary);
+ if (isSortRequired) {
+ Arrays.sort(columnWithIndexs);
+ }
+ compressMyOwnWay(extractDataAndReturnIndexes(columnWithIndexs, keyBlock));
+ if (compressData) {
+ compressDataMyOwnWay(columnWithIndexs);
+ }
+ }
+
+ /**
+ * Create an object with each column array and respective index
+ *
+ * @return
+ */
+ private ColumnWithIntIndex[] createColumnWithIndexArray(byte[][] keyBlock,
+ boolean isNoDictionary) {
+ ColumnWithIntIndex[] columnWithIndexs;
+ if (isNoDictionary) {
+ columnWithIndexs = new ColumnWithIntIndexForHighCard[keyBlock.length];
+ for (int i = 0; i < columnWithIndexs.length; i++) {
+ columnWithIndexs[i] = new ColumnWithIntIndexForHighCard(keyBlock[i], i);
+ }
+
+ } else {
+ columnWithIndexs = new ColumnWithIntIndex[keyBlock.length];
+ for (int i = 0; i < columnWithIndexs.length; i++) {
+ columnWithIndexs[i] = new ColumnWithIntIndex(keyBlock[i], i);
+ }
+ }
+
+ return columnWithIndexs;
+ }
+
+ private int[] extractDataAndReturnIndexes(ColumnWithIntIndex[] columnWithIndexs,
+ byte[][] keyBlock) {
+ int[] indexes = new int[columnWithIndexs.length];
+ for (int i = 0; i < indexes.length; i++) {
+ indexes[i] = columnWithIndexs[i].getIndex();
+ keyBlock[i] = columnWithIndexs[i].getColumn();
+ }
+ this.keyBlock = keyBlock;
+ return indexes;
+ }
+
+ /**
+ * It compresses depends up on the sequence numbers.
+ * [1,2,3,4,6,8,10,11,12,13] is translated to [1,4,6,8,10,13] and [0,6]. In
+ * first array the start and end of sequential numbers and second array
+ * keeps the indexes of where sequential numbers starts. If there is no
+ * sequential numbers then the same array it returns with empty second
+ * array.
+ *
+ * @param indexes
+ */
+ public void compressMyOwnWay(int[] indexes) {
+ List<Integer> list = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ int k = 0;
+ int i = 1;
+ for (; i < indexes.length; i++) {
+ if (indexes[i] - indexes[i - 1] == 1) {
+ k++;
+ } else {
+ if (k > 0) {
+ map.add((list.size()));
+ list.add(indexes[i - k - 1]);
+ list.add(indexes[i - 1]);
+ } else {
+ list.add(indexes[i - 1]);
+ }
+ k = 0;
+ }
+ }
+ if (k > 0) {
+ map.add((list.size()));
+ list.add(indexes[i - k - 1]);
+ list.add(indexes[i - 1]);
+ } else {
+ list.add(indexes[i - 1]);
+ }
+ dataAfterComp = convertToArray(list);
+ if (indexes.length == dataAfterComp.length) {
+ indexMap = new int[0];
+ } else {
+ indexMap = convertToArray(map);
+ }
+ if (dataAfterComp.length == 2 && indexMap.length == 1) {
+ alreadySorted = true;
+ }
+ }
+
+ private int[] convertToArray(List<Integer> list) {
+ int[] shortArray = new int[list.size()];
+ for (int i = 0; i < shortArray.length; i++) {
+ shortArray[i] = list.get(i);
+ }
+ return shortArray;
+ }
+
+ /**
+ * @return the alreadySorted
+ */
+ public boolean isAlreadySorted() {
+ return alreadySorted;
+ }
+
+ /**
+ * @return the dataAfterComp
+ */
+ public int[] getDataAfterComp() {
+ return dataAfterComp;
+ }
+
+ /**
+ * @return the indexMap
+ */
+ public int[] getIndexMap() {
+ return indexMap;
+ }
+
+ /**
+ * @return the keyBlock
+ */
+ public byte[][] getKeyBlock() {
+ return keyBlock;
+ }
+
+ private void compressDataMyOwnWay(ColumnWithIntIndex[] indexes) {
+ byte[] prvKey = indexes[0].getColumn();
+ List<ColumnWithIntIndex> list =
+ new ArrayList<ColumnWithIntIndex>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ list.add(indexes[0]);
+ int counter = 1;
+ int start = 0;
+ List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ for (int i = 1; i < indexes.length; i++) {
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, indexes[i].getColumn()) != 0) {
+ prvKey = indexes[i].getColumn();
+ list.add(indexes[i]);
+ map.add(start);
+ map.add(counter);
+ start += counter;
+ counter = 1;
+ continue;
+ }
+ counter++;
+ }
+ map.add(start);
+ map.add(counter);
+ this.keyBlock = convertToKeyArray(list);
+ if (indexes.length == keyBlock.length) {
+ dataIndexMap = new int[0];
+ } else {
+ dataIndexMap = convertToArray(map);
+ }
+ }
+
+ private byte[][] convertToKeyArray(List<ColumnWithIntIndex> list) {
+ byte[][] shortArray = new byte[list.size()][];
+ for (int i = 0; i < shortArray.length; i++) {
+ shortArray[i] = list.get(i).getColumn();
+ totalSize += shortArray[i].length;
+ }
+ return shortArray;
+ }
+
+ @Override public int[] getDataIndexMap() {
+ return dataIndexMap;
+ }
+
+ @Override public int getTotalSize() {
+ return totalSize;
+ }
+
+ @Override public byte[] getMin() {
+ return keyBlock[0];
+ }
+
+ @Override public byte[] getMax() {
+ return keyBlock[keyBlock.length - 1];
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/952cf517/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForNoInvertedIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForNoInvertedIndex.java b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForNoInvertedIndex.java
new file mode 100644
index 0000000..939d759
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/columnar/BlockIndexerStorageForNoInvertedIndex.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.columnar;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.ByteUtil;
+
+public class BlockIndexerStorageForNoInvertedIndex implements IndexStorage<int[]> {
+ private byte[][] keyBlock;
+ private byte[][] sortedBlock;
+ private int totalSize;
+ private int[] dataIndexMap;
+
+ public BlockIndexerStorageForNoInvertedIndex(byte[][] keyBlockInput, boolean compressData,
+ boolean isNoDictionary) {
+ // without invertedindex but can be RLE
+ if (compressData) {
+ // with RLE
+ byte[] prvKey = keyBlockInput[0];
+ List<byte[]> list = new ArrayList<byte[]>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ list.add(keyBlockInput[0]);
+ int counter = 1;
+ int start = 0;
+ List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ int length = keyBlockInput.length;
+ for(int i = 1; i < length; i++) {
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, keyBlockInput[i]) != 0) {
+ prvKey = keyBlockInput[i];
+ list.add(keyBlockInput[i]);
+ map.add(start);
+ map.add(counter);
+ start += counter;
+ counter = 1;
+ continue;
+ }
+ counter++;
+ }
+ map.add(start);
+ map.add(counter);
+ this.keyBlock = convertToKeyArray(list);
+ if (keyBlockInput.length == this.keyBlock.length) {
+ dataIndexMap = new int[0];
+ } else {
+ dataIndexMap = convertToArray(map);
+ }
+ } else {
+ this.keyBlock = keyBlockInput;
+ dataIndexMap = new int[0];
+ }
+
+ this.sortedBlock = new byte[keyBlock.length][];
+ System.arraycopy(keyBlock, 0, sortedBlock, 0, keyBlock.length);
+ if (isNoDictionary) {
+ Arrays.sort(sortedBlock, new Comparator<byte[]>() {
+ @Override
+ public int compare(byte[] col1, byte[] col2) {
+ return ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(col1, 2, col1.length - 2, col2, 2, col2.length - 2);
+ }
+ });
+ } else {
+ Arrays.sort(sortedBlock, new Comparator<byte[]>() {
+ @Override
+ public int compare(byte[] col1, byte[] col2) {
+ return ByteUtil.UnsafeComparer.INSTANCE.compareTo(col1, col2);
+ }
+ });
+ }
+
+ }
+
+ private int[] convertToArray(List<Integer> list) {
+ int[] shortArray = new int[list.size()];
+ for(int i = 0; i < shortArray.length; i++) {
+ shortArray[i] = list.get(i);
+ }
+ return shortArray;
+ }
+
+ private byte[][] convertToKeyArray(List<byte[]> list) {
+ byte[][] shortArray = new byte[list.size()][];
+ for (int i = 0; i < shortArray.length; i++) {
+ shortArray[i] = list.get(i);
+ totalSize += shortArray[i].length;
+ }
+ return shortArray;
+ }
+
+ @Override
+ public int[] getDataIndexMap() {
+ return dataIndexMap;
+ }
+
+ @Override
+ public int getTotalSize() {
+ return totalSize;
+ }
+
+ @Override
+ public boolean isAlreadySorted() {
+ return true;
+ }
+
+ /**
+ * no use
+ * @return
+ */
+ @Override
+ public int[] getDataAfterComp() {
+ return new int[0];
+ }
+
+ /**
+ * no use
+ * @return
+ */
+ @Override
+ public int[] getIndexMap() {
+ return new int[0];
+ }
+
+ /**
+ * @return the keyBlock
+ */
+ public byte[][] getKeyBlock() {
+ return keyBlock;
+ }
+
+ @Override public byte[] getMin() {
+ return sortedBlock[0];
+ }
+
+ @Override public byte[] getMax() {
+ return sortedBlock[sortedBlock.length - 1];
+ }
+
+}
[37/57] [abbrv] incubator-carbondata git commit: move identifier to
metadata package
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/locks/LocalFileLock.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/locks/LocalFileLock.java b/core/src/main/java/org/apache/carbondata/locks/LocalFileLock.java
deleted file mode 100644
index e61dcd5..0000000
--- a/core/src/main/java/org/apache/carbondata/locks/LocalFileLock.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.locks;
-
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.channels.FileChannel;
-import java.nio.channels.FileLock;
-import java.nio.channels.OverlappingFileLockException;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.impl.FileFactory;
-
-/**
- * This class handles the file locking in the local file system.
- * This will be handled using the file channel lock API.
- */
-public class LocalFileLock extends AbstractCarbonLock {
- /**
- * location is the location of the lock file.
- */
- private String location;
-
- /**
- * fileOutputStream of the local lock file
- */
- private FileOutputStream fileOutputStream;
-
- /**
- * channel is the FileChannel of the lock file.
- */
- private FileChannel channel;
-
- /**
- * fileLock NIO FileLock Object
- */
- private FileLock fileLock;
-
- /**
- * lock file
- */
- private String lockFile;
-
- public static final String tmpPath;
-
- private String lockFilePath;
-
- /**
- * LOGGER for logging the messages.
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(LocalFileLock.class.getName());
-
- static {
- tmpPath = System.getProperty("java.io.tmpdir");
- }
-
- /**
- * @param lockFileLocation
- * @param lockFile
- */
- public LocalFileLock(String lockFileLocation, String lockFile) {
- this.location = tmpPath + CarbonCommonConstants.FILE_SEPARATOR + lockFileLocation;
- this.lockFile = lockFile;
- initRetry();
- }
-
- /**
- * @param tableIdentifier
- * @param lockFile
- */
- public LocalFileLock(CarbonTableIdentifier tableIdentifier, String lockFile) {
- this(tableIdentifier.getDatabaseName() + CarbonCommonConstants.FILE_SEPARATOR + tableIdentifier
- .getTableName(), lockFile);
- initRetry();
- }
-
- /**
- * Lock API for locking of the file channel of the lock file.
- *
- * @return
- */
- @Override public boolean lock() {
- try {
- if (!FileFactory.isFileExist(location, FileFactory.getFileType(tmpPath))) {
- FileFactory.mkdirs(location, FileFactory.getFileType(tmpPath));
- }
- lockFilePath = location + CarbonCommonConstants.FILE_SEPARATOR +
- lockFile;
- if (!FileFactory.isFileExist(lockFilePath, FileFactory.getFileType(location))) {
- FileFactory.createNewLockFile(lockFilePath, FileFactory.getFileType(location));
- }
-
- fileOutputStream = new FileOutputStream(lockFilePath);
- channel = fileOutputStream.getChannel();
- try {
- fileLock = channel.tryLock();
- } catch (OverlappingFileLockException e) {
- return false;
- }
- if (null != fileLock) {
- return true;
- } else {
- return false;
- }
- } catch (IOException e) {
- return false;
- }
-
- }
-
- /**
- * Unlock API for unlocking of the acquired lock.
- *
- * @return
- */
- @Override public boolean unlock() {
- boolean status;
- try {
- if (null != fileLock) {
- fileLock.release();
- }
- status = true;
- } catch (IOException e) {
- status = false;
- } finally {
- if (null != fileOutputStream) {
- try {
- fileOutputStream.close();
- // deleting the lock file after releasing the lock.
- if (FileFactory.getCarbonFile(lockFilePath, FileFactory.getFileType(lockFilePath))
- .delete()) {
- LOGGER.info("Successfully deleted the lock file " + lockFilePath);
- } else {
- LOGGER.error("Not able to delete the lock file " + lockFilePath);
- }
- } catch (IOException e) {
- LOGGER.error(e.getMessage());
- }
- }
- }
- return status;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/locks/LockUsage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/locks/LockUsage.java b/core/src/main/java/org/apache/carbondata/locks/LockUsage.java
deleted file mode 100644
index b323447..0000000
--- a/core/src/main/java/org/apache/carbondata/locks/LockUsage.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.locks;
-
-/**
- * This enum is used to define the usecase of the lock.
- * Each enum value is one specific lock case.
- */
-public class LockUsage {
- public static final String LOCK = ".lock";
- public static final String METADATA_LOCK = "meta.lock";
- public static final String COMPACTION_LOCK = "compaction.lock";
- public static final String SYSTEMLEVEL_COMPACTION_LOCK = "system_level_compaction.lock";
- public static final String TABLE_STATUS_LOCK = "tablestatus.lock";
- public static final String TABLE_UPDATE_STATUS_LOCK = "tableupdatestatus.lock";
- public static final String DELETE_SEGMENT_LOCK = "delete_segment.lock";
- public static final String CLEAN_FILES_LOCK = "clean_files.lock";
- public static final String DROP_TABLE_LOCK = "droptable.lock";
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/locks/ZooKeeperLocking.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/locks/ZooKeeperLocking.java b/core/src/main/java/org/apache/carbondata/locks/ZooKeeperLocking.java
deleted file mode 100644
index 1661eb2..0000000
--- a/core/src/main/java/org/apache/carbondata/locks/ZooKeeperLocking.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.locks;
-
-import java.io.File;
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.util.CarbonProperties;
-
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.ZooDefs.Ids;
-import org.apache.zookeeper.ZooKeeper;
-
-/**
- * For Handling the zookeeper locking implementation
- */
-public class ZooKeeperLocking extends AbstractCarbonLock {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(ZooKeeperLocking.class.getName());
-
- /**
- * zk is the zookeeper client instance
- */
- private static ZooKeeper zk;
-
- /**
- * zooKeeperLocation is the location in the zoo keeper file system where the locks will be
- * maintained.
- */
- private static final String zooKeeperLocation = CarbonCommonConstants.ZOOKEEPER_LOCATION;
-
- /**
- * Unique folder for each table with DatabaseName_TableName
- */
- private final String tableIdFolder;
-
- /**
- * lockName is the name of the lock to use. This name should be same for every process that want
- * to share the same lock
- */
- private String lockName;
-
- /**
- * lockPath is the unique path created for the each instance of the carbon lock.
- */
- private String lockPath;
-
- private String lockTypeFolder;
-
- public ZooKeeperLocking(CarbonTableIdentifier tableIdentifier, String lockFile) {
- this(tableIdentifier.getDatabaseName() + File.separator + tableIdentifier.getTableName(),
- lockFile);
- }
-
- /**
- * @param lockLocation
- * @param lockFile
- */
- public ZooKeeperLocking(String lockLocation, String lockFile) {
- this.lockName = lockFile;
- this.tableIdFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + lockLocation;
-
- String zooKeeperUrl =
- CarbonProperties.getInstance().getProperty(CarbonCommonConstants.ZOOKEEPER_URL);
- zk = ZookeeperInit.getInstance(zooKeeperUrl).getZookeeper();
-
- this.lockTypeFolder = zooKeeperLocation + CarbonCommonConstants.FILE_SEPARATOR + lockLocation
- + CarbonCommonConstants.FILE_SEPARATOR + lockFile;
- try {
- createBaseNode();
- // if exists returns null then path doesnt exist. so creating.
- if (null == zk.exists(this.tableIdFolder, true)) {
- createRecursivly(this.tableIdFolder);
- }
- // if exists returns null then path doesnt exist. so creating.
- if (null == zk.exists(this.lockTypeFolder, true)) {
- zk.create(this.lockTypeFolder, new byte[1], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- }
- } catch (KeeperException | InterruptedException e) {
- LOGGER.error(e, e.getMessage());
- }
- initRetry();
- }
-
- /**
- * Creating a znode in which all the znodes (lock files )are maintained.
- */
- private void createBaseNode() throws KeeperException, InterruptedException {
- if (null == zk.exists(zooKeeperLocation, true)) {
- // creating a znode in which all the znodes (lock files )are maintained.
- zk.create(zooKeeperLocation, new byte[1], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- }
- }
-
- /**
- * Create zookeepr node if not exist
- * @param path
- * @throws KeeperException
- * @throws InterruptedException
- */
- private void createRecursivly(String path) throws KeeperException, InterruptedException {
- try {
- if (zk.exists(path, true) == null && path.length() > 0) {
- String temp = path.substring(0, path.lastIndexOf(File.separator));
- createRecursivly(temp);
- zk.create(path, null, Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- } else {
- return;
- }
- } catch (KeeperException e) {
- throw e;
- } catch (InterruptedException e) {
- throw e;
- }
-
- }
- /**
- * Handling of the locking mechanism using zoo keeper.
- */
- @Override public boolean lock() {
- try {
- // create the lock file with lockName.
- lockPath =
- zk.create(this.lockTypeFolder + CarbonCommonConstants.FILE_SEPARATOR + lockName, null,
- Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL);
-
- // get the children present in zooKeeperLocation.
- List<String> nodes = zk.getChildren(this.lockTypeFolder, null);
-
- // sort the childrens
- Collections.sort(nodes);
-
- // here the logic is , for each lock request zookeeper will create a file ending with
- // incremental digits.
- // so first request will be 00001 next is 00002 and so on.
- // if the current request is 00002 and already one previous request(00001) is present then get
- // children will give both nodes.
- // after the sort we are checking if the lock path is first or not .if it is first then lock
- // has been acquired.
-
- if (lockPath.endsWith(nodes.get(0))) {
- return true;
- } else {
- // if locking failed then deleting the created lock as next time again new lock file will be
- // created.
- zk.delete(lockPath, -1);
- return false;
- }
- } catch (KeeperException | InterruptedException e) {
- LOGGER.error(e, e.getMessage());
- return false;
- }
- }
-
- /**
- * @return status where lock file is unlocked or not.
- */
- @Override public boolean unlock() {
- try {
- // exists will return null if the path doesn't exists.
- if (null != zk.exists(lockPath, true)) {
- zk.delete(lockPath, -1);
- lockPath = null;
- }
- } catch (KeeperException | InterruptedException e) {
- LOGGER.error(e, e.getMessage());
- return false;
- }
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/main/java/org/apache/carbondata/locks/ZookeeperInit.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/locks/ZookeeperInit.java b/core/src/main/java/org/apache/carbondata/locks/ZookeeperInit.java
deleted file mode 100644
index b70579e..0000000
--- a/core/src/main/java/org/apache/carbondata/locks/ZookeeperInit.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.carbondata.locks;
-
-import java.io.IOException;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-
-import org.apache.zookeeper.WatchedEvent;
-import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.ZooKeeper;
-
-/**
- * This is a singleton class for initialization of zookeeper client.
- */
-public class ZookeeperInit {
-
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(ZookeeperInit.class.getName());
-
- private static ZookeeperInit zooKeeperInit;
- /**
- * zk is the zookeeper client instance
- */
- private ZooKeeper zk;
-
- private ZookeeperInit(String zooKeeperUrl) {
-
- int sessionTimeOut = 100000;
- try {
- zk = new ZooKeeper(zooKeeperUrl, sessionTimeOut, new DummyWatcher());
-
- } catch (IOException e) {
- LOGGER.error(e.getMessage());
- }
-
- }
-
- public static ZookeeperInit getInstance(String zooKeeperUrl) {
-
- if (null == zooKeeperInit) {
- synchronized (ZookeeperInit.class) {
- if (null == zooKeeperInit) {
- LOGGER.info("Initiating Zookeeper client.");
- zooKeeperInit = new ZookeeperInit(zooKeeperUrl);
- }
- }
- }
- return zooKeeperInit;
-
- }
-
- public static ZookeeperInit getInstance() {
- return zooKeeperInit;
- }
-
- public ZooKeeper getZookeeper() {
- return zk;
- }
-
- private static class DummyWatcher implements Watcher {
- public void process(WatchedEvent event) {
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/test/java/org/apache/carbondata/core/AbsoluteTableIdentifierTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/AbsoluteTableIdentifierTest.java b/core/src/test/java/org/apache/carbondata/core/AbsoluteTableIdentifierTest.java
index aaf0d97..42a5057 100644
--- a/core/src/test/java/org/apache/carbondata/core/AbsoluteTableIdentifierTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/AbsoluteTableIdentifierTest.java
@@ -19,8 +19,8 @@
package org.apache.carbondata.core;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.junit.BeforeClass;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/test/java/org/apache/carbondata/core/CarbonTableIdentifierTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/CarbonTableIdentifierTest.java b/core/src/test/java/org/apache/carbondata/core/CarbonTableIdentifierTest.java
index 942291c..bf0402d 100644
--- a/core/src/test/java/org/apache/carbondata/core/CarbonTableIdentifierTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/CarbonTableIdentifierTest.java
@@ -18,7 +18,7 @@
*/
package org.apache.carbondata.core;
-import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.junit.BeforeClass;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/test/java/org/apache/carbondata/core/ColumnIdentifierTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/ColumnIdentifierTest.java b/core/src/test/java/org/apache/carbondata/core/ColumnIdentifierTest.java
index f4e7094..4ce3cd5 100644
--- a/core/src/test/java/org/apache/carbondata/core/ColumnIdentifierTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/ColumnIdentifierTest.java
@@ -18,7 +18,7 @@
*/
package org.apache.carbondata.core;
-import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.metadata.DataType;
import org.junit.BeforeClass;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
index 4fd4d37..30bfdd5 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCacheTest.java
@@ -32,8 +32,8 @@ import java.util.List;
import java.util.Properties;
import org.apache.carbondata.core.cache.Cache;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImplTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImplTest.java
index cae1b06..dbfc628 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryCacheLoaderImplTest.java
@@ -26,8 +26,8 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.reader.CarbonDictionaryReaderImpl;
import org.apache.carbondata.core.reader.sortindex.CarbonDictionarySortIndexReaderImpl;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifierTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifierTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifierTest.java
index f5e9854..2fc8bcd 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifierTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/DictionaryColumnUniqueIdentifierTest.java
@@ -21,8 +21,8 @@ package org.apache.carbondata.core.cache.dictionary;
import mockit.Mock;
import mockit.MockUp;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.metadata.DataType;
import org.junit.BeforeClass;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCacheTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCacheTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCacheTest.java
index 020f65a..c6fda5c 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCacheTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ForwardDictionaryCacheTest.java
@@ -30,8 +30,8 @@ import java.util.UUID;
import org.apache.carbondata.core.cache.Cache;
import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.cache.CacheType;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.writer.sortindex.CarbonDictionarySortIndexWriter;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCacheTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCacheTest.java b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCacheTest.java
index 7bb60aa..c0310d7 100644
--- a/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCacheTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCacheTest.java
@@ -34,8 +34,8 @@ import mockit.MockUp;
import org.apache.carbondata.core.cache.Cache;
import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.cache.CacheType;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.reader.CarbonDictionaryColumnMetaChunk;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/test/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStoreTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStoreTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStoreTest.java
index 56ed162..180c82f 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStoreTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/SegmentTaskIndexStoreTest.java
@@ -27,9 +27,9 @@ import java.util.Map;
import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.cache.CacheType;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.datastore.block.AbstractIndex;
import org.apache.carbondata.core.datastore.block.SegmentTaskIndex;
import org.apache.carbondata.core.datastore.block.SegmentTaskIndexWrapper;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/test/java/org/apache/carbondata/core/datastore/block/BlockInfoTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/block/BlockInfoTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/block/BlockInfoTest.java
index e71d4f9..f0e16c5 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/block/BlockInfoTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/block/BlockInfoTest.java
@@ -18,7 +18,7 @@
*/
package org.apache.carbondata.core.datastore.block;
-import org.apache.carbondata.core.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.junit.BeforeClass;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/test/java/org/apache/carbondata/core/datastore/block/TableBlockInfoTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/block/TableBlockInfoTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/block/TableBlockInfoTest.java
index 21270e7..3592a16 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/block/TableBlockInfoTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/block/TableBlockInfoTest.java
@@ -21,7 +21,7 @@ package org.apache.carbondata.core.datastore.block;
import mockit.Mock;
import mockit.MockUp;
-import org.apache.carbondata.core.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.junit.BeforeClass;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/test/java/org/apache/carbondata/core/datastore/block/TableTaskInfoTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/datastore/block/TableTaskInfoTest.java b/core/src/test/java/org/apache/carbondata/core/datastore/block/TableTaskInfoTest.java
index c64c33f..c2229cf 100644
--- a/core/src/test/java/org/apache/carbondata/core/datastore/block/TableTaskInfoTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/datastore/block/TableTaskInfoTest.java
@@ -18,7 +18,7 @@
*/
package org.apache.carbondata.core.datastore.block;
-import org.apache.carbondata.core.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.junit.BeforeClass;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/test/java/org/apache/carbondata/core/path/CarbonFormatDirectoryStructureTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/path/CarbonFormatDirectoryStructureTest.java b/core/src/test/java/org/apache/carbondata/core/path/CarbonFormatDirectoryStructureTest.java
index 684d48d..50c68bb 100644
--- a/core/src/test/java/org/apache/carbondata/core/path/CarbonFormatDirectoryStructureTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/path/CarbonFormatDirectoryStructureTest.java
@@ -22,9 +22,8 @@ package org.apache.carbondata.core.path;
import java.io.IOException;
import java.util.UUID;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.path.CarbonStorePath;
-import org.apache.carbondata.core.path.CarbonTablePath;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+
import org.junit.Test;
import static junit.framework.TestCase.assertTrue;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/test/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImplTest.java b/core/src/test/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImplTest.java
index 59ac840..5432ad6 100644
--- a/core/src/test/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/reader/CarbonDictionaryReaderImplTest.java
@@ -27,8 +27,8 @@ import java.util.UUID;
import org.apache.carbondata.core.service.impl.PathFactory;
import org.apache.carbondata.core.service.CarbonCommonFactory;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.service.PathService;
import mockit.Mock;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/test/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java b/core/src/test/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java
index e1e0e94..e0708dc 100644
--- a/core/src/test/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/reader/sortindex/CarbonDictionarySortIndexReaderImplTest.java
@@ -24,8 +24,8 @@ import java.util.Arrays;
import java.util.List;
import java.util.UUID;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.datastorage.filesystem.CarbonFile;
import org.apache.carbondata.core.datastorage.impl.FileFactory;
import org.apache.carbondata.core.util.CarbonUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
index 1da4415..72fa8af 100644
--- a/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/scan/filter/FilterUtilTest.java
@@ -23,8 +23,8 @@ import java.util.List;
import java.util.UUID;
import org.apache.carbondata.core.cache.dictionary.AbstractDictionaryCacheTest;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.datastore.IndexKey;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.metadata.DataType;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/test/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLockTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLockTest.java b/core/src/test/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLockTest.java
new file mode 100644
index 0000000..d69b6be
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/updatestatus/locks/LocalFileLockTest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.updatestatus.locks;
+
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.pentaho.di.core.util.Assert;
+
+/**
+ * Test class to test the functionality of the local file locking.
+ *
+ * @author Administrator
+ */
+public class LocalFileLockTest {
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @Before public void setUp() throws Exception {
+ }
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @After public void tearDown() throws Exception {
+ }
+
+ @Test public void testingLocalFileLockingByAcquiring2Locks() {
+
+ CarbonTableIdentifier carbonTableIdentifier = new CarbonTableIdentifier("databaseName", "tableName", "tableId");
+ LocalFileLock localLock1 =
+ new LocalFileLock(carbonTableIdentifier,
+ LockUsage.METADATA_LOCK);
+ Assert.assertTrue(localLock1.lock());
+ LocalFileLock localLock2 =
+ new LocalFileLock(carbonTableIdentifier,
+ LockUsage.METADATA_LOCK);
+ Assert.assertTrue(!localLock2.lock());
+
+ Assert.assertTrue(localLock1.unlock());
+ Assert.assertTrue(localLock2.lock());
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/test/java/org/apache/carbondata/core/updatestatus/locks/ZooKeeperLockingTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/updatestatus/locks/ZooKeeperLockingTest.java b/core/src/test/java/org/apache/carbondata/core/updatestatus/locks/ZooKeeperLockingTest.java
new file mode 100644
index 0000000..f0037cf
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/updatestatus/locks/ZooKeeperLockingTest.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.updatestatus.locks;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.util.Properties;
+
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+import mockit.NonStrictExpectations;
+import org.apache.zookeeper.server.ServerConfig;
+import org.apache.zookeeper.server.ZooKeeperServerMain;
+import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * @author Administrator
+ */
+public class ZooKeeperLockingTest {
+
+ int freePort;
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @Before public void setUp() throws Exception {
+ Properties startupProperties = new Properties();
+ startupProperties.setProperty("dataDir", (new File("./target").getAbsolutePath()));
+ startupProperties.setProperty("dataLogDir", (new File("./target").getAbsolutePath()));
+ freePort = findFreePort();
+ startupProperties.setProperty("clientPort", "" + freePort);
+ QuorumPeerConfig quorumConfiguration = new QuorumPeerConfig();
+ try {
+ quorumConfiguration.parseProperties(startupProperties);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ final ZooKeeperServerMain zooKeeperServer = new ZooKeeperServerMain();
+ final ServerConfig configuration = new ServerConfig();
+ configuration.readFrom(quorumConfiguration);
+ new Thread() {
+ public void run() {
+ try {
+ zooKeeperServer.runFromConfig(configuration);
+ } catch (IOException e) {
+ System.out.println("ZooKeeper failure");
+ }
+ }
+ }.start();
+ }
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @After public void tearDown() throws Exception {
+ }
+
+ @Test public void testZooKeeperLockingByTryingToAcquire2Locks()
+ throws IllegalArgumentException, IllegalAccessException, NoSuchFieldException,
+ SecurityException {
+
+ final CarbonProperties cp = CarbonProperties.getInstance();
+ new NonStrictExpectations(cp) {
+ {
+ cp.getProperty("/CarbonLocks");
+ result = "/carbontests";
+ cp.getProperty("spark.deploy.zookeeper.url");
+ result = "127.0.0.1:" + freePort;
+ }
+ };
+
+ ZookeeperInit zki = ZookeeperInit.getInstance("127.0.0.1:" + freePort);
+
+ CarbonTableIdentifier tableIdentifier = new CarbonTableIdentifier("dbName", "tableName", "tableId");
+ ZooKeeperLocking zkl =
+ new ZooKeeperLocking(tableIdentifier,
+ LockUsage.METADATA_LOCK);
+ Assert.assertTrue(zkl.lock());
+
+ ZooKeeperLocking zk2 = new ZooKeeperLocking(
+ tableIdentifier, LockUsage.METADATA_LOCK);
+ Assert.assertTrue(!zk2.lock());
+
+ Assert.assertTrue(zkl.unlock());
+ Assert.assertTrue(zk2.lock());
+ Assert.assertTrue(zk2.unlock());
+ }
+
+ /**
+ * For finding the free port available.
+ *
+ * @return
+ */
+ private static int findFreePort() {
+ ServerSocket socket = null;
+ try {
+ socket = new ServerSocket(0);
+ socket.setReuseAddress(true);
+ int port = socket.getLocalPort();
+ try {
+ socket.close();
+ } catch (IOException e) {
+ // Ignore IOException on close()
+ }
+ return port;
+ } catch (Exception e) {
+ // Ignore
+ } finally {
+ if (socket != null) {
+ try {
+ socket.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ return 2181;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
index 7f6edd1..37f1ef6 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
@@ -21,7 +21,7 @@ package org.apache.carbondata.core.util;
import mockit.Mock;
import mockit.MockUp;
-import org.apache.carbondata.core.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java b/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java
index f1ee0a0..f1b8070 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/DataFileFooterConverterTest.java
@@ -22,7 +22,7 @@ package org.apache.carbondata.core.util;
import mockit.Mock;
import mockit.MockUp;
-import org.apache.carbondata.core.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.metadata.blocklet.SegmentInfo;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java b/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java
index 6037ee8..4994942 100644
--- a/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/writer/CarbonDictionaryWriterImplTest.java
@@ -34,8 +34,8 @@ import java.util.List;
import java.util.Properties;
import java.util.UUID;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImplTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImplTest.java b/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImplTest.java
index 2f5841c..2b689be 100644
--- a/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImplTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImplTest.java
@@ -24,8 +24,8 @@ import java.util.Arrays;
import java.util.List;
import java.util.UUID;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.reader.sortindex.CarbonDictionarySortIndexReader;
import org.apache.carbondata.core.reader.sortindex.CarbonDictionarySortIndexReaderImpl;
import org.apache.carbondata.core.util.CarbonUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/examples/spark/src/main/scala/org/apache/carbondata/examples/GenerateDictionaryExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/GenerateDictionaryExample.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/GenerateDictionaryExample.scala
index 148940a..1a0e9bc 100644
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/GenerateDictionaryExample.scala
+++ b/examples/spark/src/main/scala/org/apache/carbondata/examples/GenerateDictionaryExample.scala
@@ -19,11 +19,11 @@ package org.apache.carbondata.examples
import org.apache.spark.sql.{CarbonContext, CarbonEnv, CarbonRelation}
-import org.apache.carbondata.core.CarbonTableIdentifier
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.path.CarbonStorePath
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier
import org.apache.carbondata.examples.util.ExampleUtils
import org.apache.carbondata.spark.load.CarbonLoaderUtil
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
index 23429c4..362510e 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
@@ -23,8 +23,8 @@ import java.lang.reflect.Constructor;
import java.util.*;
import org.apache.carbondata.core.update.data.BlockMappingVO;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.datastore.DataRefNodeFinder;
import org.apache.carbondata.core.datastore.IndexKey;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
index a42995f..115fc7d 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
@@ -27,7 +27,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.carbondata.core.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
import org.apache.carbondata.core.datastore.block.BlockletInfos;
import org.apache.carbondata.core.datastore.block.Distributable;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonMultiBlockSplit.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonMultiBlockSplit.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonMultiBlockSplit.java
index ff30b2f..06118ab 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonMultiBlockSplit.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonMultiBlockSplit.java
@@ -25,7 +25,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.InputSplit;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
index a036b2a..221700c 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
@@ -26,7 +26,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.datastore.DataRefNode;
import org.apache.carbondata.core.datastore.DataRefNodeFinder;
import org.apache.carbondata.core.datastore.IndexKey;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/CarbonReadSupport.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/CarbonReadSupport.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/CarbonReadSupport.java
index d96ad4c..0a99615 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/CarbonReadSupport.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/CarbonReadSupport.java
@@ -20,7 +20,7 @@ package org.apache.carbondata.hadoop.readsupport;
import java.io.IOException;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/AbstractDictionaryDecodedReadSupport.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/AbstractDictionaryDecodedReadSupport.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/AbstractDictionaryDecodedReadSupport.java
index c561ec4..6ce7495 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/AbstractDictionaryDecodedReadSupport.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/AbstractDictionaryDecodedReadSupport.java
@@ -25,7 +25,7 @@ import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.cache.CacheType;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.metadata.Encoding;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/ArrayWritableReadSupport.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/ArrayWritableReadSupport.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/ArrayWritableReadSupport.java
index bf48952..c787d3e 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/ArrayWritableReadSupport.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/ArrayWritableReadSupport.java
@@ -18,7 +18,7 @@
*/
package org.apache.carbondata.hadoop.readsupport.impl;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.hadoop.readsupport.CarbonReadSupport;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/RawDataReadSupport.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/RawDataReadSupport.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/RawDataReadSupport.java
index b08b665..479c066 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/RawDataReadSupport.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/readsupport/impl/RawDataReadSupport.java
@@ -18,7 +18,7 @@
*/
package org.apache.carbondata.hadoop.readsupport.impl;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
import org.apache.carbondata.hadoop.readsupport.CarbonReadSupport;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
index 7585032..790a42c 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/CarbonInputFormatUtil.java
@@ -22,7 +22,7 @@ package org.apache.carbondata.hadoop.util;
import java.io.IOException;
import java.util.List;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
index 9b95f0a..4450339 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/SchemaReader.java
@@ -20,7 +20,7 @@ package org.apache.carbondata.hadoop.util;
import java.io.IOException;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.converter.SchemaConverter;
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonInputMapperTest.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonInputMapperTest.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonInputMapperTest.java
index def162c..5bbbb83 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonInputMapperTest.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonInputMapperTest.java
@@ -25,7 +25,7 @@ import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.DataType;
import org.apache.carbondata.core.util.CarbonUtil;
import org.apache.carbondata.hadoop.CarbonInputFormat;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
index fa3710f..384e7c6 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
@@ -26,10 +26,10 @@ import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.cache.CacheType;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.CarbonDataLoadSchema;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.processing.model.CarbonDataLoadSchema;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.converter.SchemaConverter;
import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl;
@@ -59,9 +59,9 @@ import org.apache.carbondata.core.writer.sortindex.CarbonDictionarySortIndexWrit
import org.apache.carbondata.core.writer.sortindex.CarbonDictionarySortIndexWriterImpl;
import org.apache.carbondata.core.writer.sortindex.CarbonDictionarySortInfo;
import org.apache.carbondata.core.writer.sortindex.CarbonDictionarySortInfoPreparator;
-import org.apache.carbondata.fileoperations.AtomicFileOperations;
-import org.apache.carbondata.fileoperations.AtomicFileOperationsImpl;
-import org.apache.carbondata.fileoperations.FileWriteOperation;
+import org.apache.carbondata.core.atomic.AtomicFileOperations;
+import org.apache.carbondata.core.atomic.AtomicFileOperationsImpl;
+import org.apache.carbondata.core.atomic.FileWriteOperation;
import org.apache.carbondata.processing.api.dataloader.DataLoadModel;
import org.apache.carbondata.processing.api.dataloader.SchemaInfo;
import org.apache.carbondata.processing.constants.TableOptionConstant;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestCreateTableWithDouble.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestCreateTableWithDouble.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestCreateTableWithDouble.scala
index 5691605..01a4c66 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestCreateTableWithDouble.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestCreateTableWithDouble.scala
@@ -23,9 +23,9 @@ import org.apache.spark.sql.Row
import org.apache.spark.sql.common.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.CarbonTableIdentifier
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier
/**
* Test class of creating and loading for carbon table with double
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBoundaryConditionsTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBoundaryConditionsTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBoundaryConditionsTest.scala
index 246f779..487dab8 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBoundaryConditionsTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBoundaryConditionsTest.scala
@@ -21,8 +21,8 @@ package org.apache.carbondata.spark.testsuite.datacompaction
import org.apache.spark.sql.common.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.CarbonTableIdentifier
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier
import org.apache.carbondata.core.util.CarbonProperties
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionCardinalityBoundryTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionCardinalityBoundryTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionCardinalityBoundryTest.scala
index 47976b7..5cc7af9 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionCardinalityBoundryTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionCardinalityBoundryTest.scala
@@ -24,8 +24,9 @@ import org.apache.spark.sql.Row
import org.apache.spark.sql.common.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.{AbsoluteTableIdentifier, CarbonTableIdentifier}
+import org.apache.carbondata.core.CarbonTableIdentifier
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.core.updatestatus.SegmentStatusManager
import org.apache.carbondata.core.util.CarbonProperties
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionLockTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionLockTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionLockTest.scala
index 4e7f6da..83753c5 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionLockTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionLockTest.scala
@@ -25,12 +25,13 @@ import scala.collection.JavaConverters._
import org.apache.spark.sql.common.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.{AbsoluteTableIdentifier, CarbonTableIdentifier}
+import org.apache.carbondata.core.CarbonTableIdentifier
import org.apache.carbondata.core.path.{CarbonStorePath, CarbonTablePath}
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.core.updatestatus.SegmentStatusManager
import org.apache.carbondata.core.util.CarbonProperties
-import org.apache.carbondata.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
+import org.apache.carbondata.core.updatestatus.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
index 8e29471..97d5fb0 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionIgnoreInMinorTest.scala
@@ -23,11 +23,12 @@ import scala.collection.JavaConverters._
import org.apache.spark.sql.common.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.{AbsoluteTableIdentifier, CarbonTableIdentifier}
+import org.apache.carbondata.core.CarbonTableIdentifier
import org.apache.carbondata.core.datastore.TableSegmentUniqueIdentifier
import org.apache.carbondata.core.datastore.block.SegmentTaskIndexWrapper
import org.apache.carbondata.core.path.CarbonStorePath
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.core.updatestatus.SegmentStatusManager
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.hadoop.CacheClient
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionStopsAfterCompaction.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionStopsAfterCompaction.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionStopsAfterCompaction.scala
index 6691f49..ce8f547 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionStopsAfterCompaction.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/MajorCompactionStopsAfterCompaction.scala
@@ -23,8 +23,9 @@ import scala.collection.JavaConverters._
import org.apache.spark.sql.common.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.{AbsoluteTableIdentifier, CarbonTableIdentifier}
+import org.apache.carbondata.core.CarbonTableIdentifier
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.core.updatestatus.SegmentStatusManager
import org.apache.carbondata.core.util.CarbonProperties
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
index 92b7c71..76e815f 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataretention/DataRetentionTestCase.scala
@@ -22,16 +22,17 @@ package org.apache.carbondata.spark.testsuite.dataretention
import java.text.SimpleDateFormat
import org.apache.carbondata.core.updatestatus.SegmentStatusManager
-import org.apache.carbondata.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
+import org.apache.carbondata.core.updatestatus.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
import org.apache.commons.lang3.time.DateUtils
import org.apache.spark.sql.Row
import org.apache.spark.sql.common.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.{AbsoluteTableIdentifier, CarbonTableIdentifier}
+import org.apache.carbondata.core.CarbonTableIdentifier
import org.apache.carbondata.core.path.CarbonStorePath
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.load.LoadMetadataDetails
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier}
import org.apache.carbondata.core.util.CarbonProperties
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
index fbcccda..9c80d7c 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
@@ -32,9 +32,9 @@ import org.apache.carbondata.core.cache.CacheProvider;
import org.apache.carbondata.core.cache.CacheType;
import org.apache.carbondata.core.cache.dictionary.Dictionary;
import org.apache.carbondata.core.cache.dictionary.DictionaryColumnUniqueIdentifier;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.CarbonTableIdentifier;
-import org.apache.carbondata.core.ColumnIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnIdentifier;
import org.apache.carbondata.core.datastore.block.Distributable;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.DataType;
@@ -51,10 +51,10 @@ import org.apache.carbondata.core.update.CarbonUpdateUtil;
import org.apache.carbondata.core.updatestatus.SegmentStatusManager;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.fileoperations.AtomicFileOperations;
-import org.apache.carbondata.fileoperations.AtomicFileOperationsImpl;
-import org.apache.carbondata.fileoperations.FileWriteOperation;
-import org.apache.carbondata.locks.ICarbonLock;
+import org.apache.carbondata.core.atomic.AtomicFileOperations;
+import org.apache.carbondata.core.atomic.AtomicFileOperationsImpl;
+import org.apache.carbondata.core.atomic.FileWriteOperation;
+import org.apache.carbondata.core.updatestatus.locks.ICarbonLock;
import org.apache.carbondata.processing.api.dataloader.DataLoadModel;
import org.apache.carbondata.processing.api.dataloader.SchemaInfo;
import org.apache.carbondata.processing.csvload.DataGraphExecuter;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java
index 16dfcba..3c18f64 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java
@@ -35,7 +35,7 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionUtil.java
index e9bd4ae..fb19387 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonCompactionUtil.java
@@ -26,7 +26,7 @@ import java.util.Map;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.block.TaskBlockInfo;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
index 5523dcf..7ef4cec 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
@@ -27,8 +27,8 @@ import java.util.*;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.path.CarbonStorePath;
import org.apache.carbondata.core.path.CarbonTablePath;
@@ -45,9 +45,9 @@ import org.apache.carbondata.core.updatestatus.SegmentStatusManager;
import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager;
import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.core.writer.CarbonDeleteDeltaWriterImpl;
-import org.apache.carbondata.locks.CarbonLockFactory;
-import org.apache.carbondata.locks.ICarbonLock;
-import org.apache.carbondata.locks.LockUsage;
+import org.apache.carbondata.core.updatestatus.locks.CarbonLockFactory;
+import org.apache.carbondata.core.updatestatus.locks.ICarbonLock;
+import org.apache.carbondata.core.updatestatus.locks.LockUsage;
import org.apache.carbondata.processing.model.CarbonLoadModel;
import org.apache.carbondata.spark.load.CarbonLoaderUtil;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/RowResultMerger.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/RowResultMerger.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/RowResultMerger.java
index 825c98d..3802ed2 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/RowResultMerger.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/RowResultMerger.java
@@ -23,7 +23,7 @@ import java.util.*;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.datastore.block.SegmentProperties;
import org.apache.carbondata.core.metadata.CarbonMetadata;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/TableMeta.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/TableMeta.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/TableMeta.java
index f88c678..887c06c 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/TableMeta.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/TableMeta.java
@@ -21,7 +21,7 @@ package org.apache.carbondata.spark.merger;
import java.io.Serializable;
-import org.apache.carbondata.core.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
public class TableMeta implements Serializable {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
index a3bdfc6..df3ee75 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
@@ -27,8 +27,7 @@ import org.apache.spark.sql.catalyst.expressions.{Cast, Literal}
import org.apache.spark.sql.types.TimestampType
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.AbsoluteTableIdentifier
-import org.apache.carbondata.core.metadata.CarbonMetadata
+import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata}
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.updatestatus.SegmentStatusManager
import org.apache.carbondata.spark.exception.MalformedCarbonCommandException
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonSparkFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonSparkFactory.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonSparkFactory.scala
index 7cc1dcb..89cbe41 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonSparkFactory.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonSparkFactory.scala
@@ -16,8 +16,8 @@
*/
package org.apache.carbondata.spark
-import org.apache.carbondata.core.{CarbonTableIdentifier, ColumnIdentifier}
import org.apache.carbondata.core.metadata.schema.table.column.{CarbonDimension, ColumnSchema}
+import org.apache.carbondata.core.metadata.{CarbonTableIdentifier, ColumnIdentifier}
/**
* Column validator
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
index bd8e54c..9ebda94 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/DictionaryDetailHelper.scala
@@ -22,8 +22,8 @@ import scala.collection.mutable.HashMap
import org.apache.carbondata.core.datastorage.filesystem.{CarbonFile, CarbonFileFilter}
import org.apache.carbondata.core.datastorage.impl.FileFactory
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
+import org.apache.carbondata.core.metadata.{CarbonTableIdentifier, ColumnIdentifier}
import org.apache.carbondata.core.path.{CarbonStorePath, CarbonTablePath}
-import org.apache.carbondata.core.{CarbonTableIdentifier, ColumnIdentifier}
class DictionaryDetailHelper extends DictionaryDetailService {
def getDictionaryDetail(dictfolderPath: String, primDimensions: Array[CarbonDimension],
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d0d25c9b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
index 045ed2f..c6247d0 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
@@ -33,15 +33,16 @@ import org.apache.spark.rdd.RDD
import org.apache.spark.sql.Row
import org.apache.carbondata.common.logging.LogServiceFactory
-import org.apache.carbondata.core.{CarbonTableIdentifier, ColumnIdentifier}
+import org.apache.carbondata.core.metadata.ColumnIdentifier
import org.apache.carbondata.core.cache.dictionary.Dictionary
import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
import org.apache.carbondata.core.path.CarbonTablePath
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datastorage.impl.FileFactory
+import org.apache.carbondata.core.metadata.{CarbonTableIdentifier, ColumnIdentifier}
import org.apache.carbondata.core.service.{CarbonCommonFactory, PathService}
import org.apache.carbondata.core.util.{CarbonProperties, CarbonTimeStatisticsFactory, CarbonUtil}
-import org.apache.carbondata.locks.{CarbonLockFactory, LockUsage}
+import org.apache.carbondata.core.updatestatus.locks.{CarbonLockFactory, LockUsage}
import org.apache.carbondata.processing.model.CarbonLoadModel
import org.apache.carbondata.spark.load.CarbonLoaderUtil
import org.apache.carbondata.spark.tasks.{DictionaryWriterTask, SortIndexWriterTask}