You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/01/08 14:30:23 UTC
[5/7] incubator-carbondata git commit: Added support for offheap
storage in query
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureChunkStoreFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureChunkStoreFactory.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureChunkStoreFactory.java
new file mode 100644
index 0000000..e7068b0
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureChunkStoreFactory.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe.SafeByteMeasureChunkStore;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe.SafeDoubleMeasureChunkStore;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe.SafeIntMeasureChunkStore;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe.SafeLongMeasureChunkStore;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe.SafeShortMeasureChunkStore;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe.UnsafeByteMeasureChunkStore;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe.UnsafeDoubleMeasureChunkStore;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe.UnsafeIntMeasureChunkStore;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe.UnsafeLongMeasureChunkStore;
+import org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe.UnsafeShortMeasureChunkStore;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.CarbonProperties;
+import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+
+/**
+ * Factory class for getting the measure store type
+ */
+public class MeasureChunkStoreFactory {
+
+ /**
+ * instance type
+ */
+ public static final MeasureChunkStoreFactory INSTANCE = new MeasureChunkStoreFactory();
+
+ /**
+ * is unsafe
+ */
+ private static final boolean isUnsafe;
+
+ static {
+ isUnsafe = Boolean.parseBoolean(CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.ENABLE_UNSAFE_IN_QUERY_EXECUTION,
+ CarbonCommonConstants.ENABLE_UNSAFE_IN_QUERY_EXECUTION_DEFAULTVALUE));
+ }
+
+ private MeasureChunkStoreFactory() {
+ }
+
+ /**
+ * Below method will be used to get the measure data chunk store based on data type
+ *
+ * @param dataType data type
+ * @param numberOfRows number of rows
+ * @return measure chunk store
+ */
+ public MeasureDataChunkStore getMeasureDataChunkStore(DataType dataType, int numberOfRows) {
+ if (isUnsafe) {
+ switch (dataType) {
+ case DATA_BYTE:
+ return new SafeByteMeasureChunkStore(numberOfRows);
+ case DATA_SHORT:
+ return new SafeShortMeasureChunkStore(numberOfRows);
+ case DATA_INT:
+ return new SafeIntMeasureChunkStore(numberOfRows);
+ case DATA_LONG:
+ return new SafeLongMeasureChunkStore(numberOfRows);
+ case DATA_DOUBLE:
+ return new SafeDoubleMeasureChunkStore(numberOfRows);
+ default:
+ return new SafeDoubleMeasureChunkStore(numberOfRows);
+ }
+ } else {
+ switch (dataType) {
+ case DATA_BYTE:
+ return new UnsafeByteMeasureChunkStore(numberOfRows);
+ case DATA_SHORT:
+ return new UnsafeShortMeasureChunkStore(numberOfRows);
+ case DATA_INT:
+ return new UnsafeIntMeasureChunkStore(numberOfRows);
+ case DATA_LONG:
+ return new UnsafeLongMeasureChunkStore(numberOfRows);
+ case DATA_DOUBLE:
+ return new UnsafeDoubleMeasureChunkStore(numberOfRows);
+ default:
+ return new UnsafeDoubleMeasureChunkStore(numberOfRows);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureDataChunkStore.java
new file mode 100644
index 0000000..fed4ab4
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/MeasureDataChunkStore.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store;
+
+import java.math.BigDecimal;
+
+/**
+ * Responsibility is store the measure data in memory,
+ * memory can be on heap or offheap based on the user configuration
+ */
+public interface MeasureDataChunkStore<T> {
+
+ /**
+ * Below method will be used to put the data to memory
+ *
+ * @param data
+ */
+ void putData(T data);
+
+ /**
+ * to get byte value
+ *
+ * @param index
+ * @return byte value based on index
+ */
+ byte getByte(int index);
+
+ /**
+ * to get the short value
+ *
+ * @param index
+ * @return short value based on index
+ */
+ short getShort(int index);
+
+ /**
+ * to get the int value
+ *
+ * @param index
+ * @return int value based on index
+ */
+ int getInt(int index);
+
+ /**
+ * to get the long value
+ *
+ * @param index
+ * @return long value based on index
+ */
+ long getLong(int index);
+
+ /**
+ * to get the double value
+ *
+ * @param index
+ * @return double value based on index
+ */
+ double getDouble(int index);
+
+ /**
+ * To get the bigdecimal value
+ *
+ * @param index
+ * @return bigdecimal value based on index
+ */
+ BigDecimal getBigDecimal(int index);
+
+ /**
+ * To free the occupied memory
+ */
+ void freeMemory();
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java
new file mode 100644
index 0000000..0ed2218
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbsractDimensionDataChunkStore.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.store.DimensionDataChunkStore;
+
+/**
+ * Responsibility is to store dimension data
+ */
+public abstract class SafeAbsractDimensionDataChunkStore implements DimensionDataChunkStore {
+
+ /**
+ * data chunk for dimension column
+ */
+ protected byte[] data;
+
+ /**
+ * inverted index
+ */
+ protected int[] invertedIndex;
+
+ /**
+ * inverted index reverser
+ */
+ protected int[] invertedIndexReverse;
+
+ /**
+ * to check whether dimension column was explicitly sorted or not
+ */
+ protected boolean isExplictSorted;
+
+ /**
+ * Constructor
+ *
+ * @param totalSize total size of the data to be kept
+ * @param isInvertedIdex is inverted index present
+ * @param numberOfRows total number of rows
+ */
+ public SafeAbsractDimensionDataChunkStore(boolean isInvertedIdex) {
+ this.isExplictSorted = isInvertedIdex;
+ }
+
+ /**
+ * Below method will be used to put the rows and its metadata in offheap
+ *
+ * @param invertedIndex inverted index to be stored
+ * @param invertedIndexReverse inverted index reverse to be stored
+ * @param data data to be stored
+ */
+ @Override public void putArray(final int[] invertedIndex, final int[] invertedIndexReverse,
+ final byte[] data) {
+ this.data = data;
+ this.invertedIndex = invertedIndex;
+ this.invertedIndexReverse = invertedIndexReverse;
+ }
+
+ /**
+ * Below method will be used to free the memory occupied by the column chunk
+ */
+ @Override public void freeMemory() {
+ // do nothing as GC will take care of freeing memory
+ }
+
+ /**
+ * Below method will be used to get the inverted index
+ *
+ * @param rowId row id
+ * @return inverted index based on row id passed
+ */
+ @Override public int getInvertedIndex(int rowId) {
+ return invertedIndex[rowId];
+ }
+
+ /**
+ * Below method will be used to get the surrogate key of the based on the row
+ * id passed
+ *
+ * @param rowId row id
+ * @return surrogate key
+ */
+ @Override public int getSurrogate(int rowId) {
+ throw new UnsupportedOperationException("Operation not supported");
+ }
+
+ /**
+ * @return size of each column value
+ */
+ @Override public int getColumnValueSize() {
+ throw new UnsupportedOperationException("Operation not supported");
+ }
+
+ /**
+ * @return whether column was explicitly sorted or not
+ */
+ @Override public boolean isExplicitSorted() {
+ return isExplictSorted;
+ }
+
+ /**
+ * Below method will be used to fill the row values to data array
+ *
+ * @param rowId row id of the data to be filled
+ * @param data buffer in which data will be filled
+ * @param offset off the of the buffer
+ */
+ @Override public void fillRow(int rowId, byte[] data, int offset) {
+ throw new UnsupportedOperationException("Operation not supported");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbstractMeasureDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbstractMeasureDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbstractMeasureDataChunkStore.java
new file mode 100644
index 0000000..0daeaa4
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeAbstractMeasureDataChunkStore.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe;
+
+import java.math.BigDecimal;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureDataChunkStore;
+
+/**
+ * Responsibility is store the measure data in memory,
+ */
+public abstract class SafeAbstractMeasureDataChunkStore<T> implements
+ MeasureDataChunkStore<T> {
+
+ /**
+ * number of rows
+ */
+ protected int numberOfRows;
+
+ public SafeAbstractMeasureDataChunkStore(int numberOfRows) {
+ this.numberOfRows = numberOfRows;
+ }
+
+ /**
+ * to get the byte value
+ *
+ * @param index
+ * @return byte value based on index
+ */
+ @Override
+ public byte getByte(int index) {
+ throw new UnsupportedOperationException("Operation not supported");
+ }
+
+ /**
+ * to get the short value
+ *
+ * @param index
+ * @return short value based on index
+ */
+ @Override
+ public short getShort(int index) {
+ throw new UnsupportedOperationException("Operation not supported");
+ }
+
+ /**
+ * to get the int value
+ *
+ * @param index
+ * @return int value based on index
+ */
+ @Override
+ public int getInt(int index) {
+ throw new UnsupportedOperationException("Operation not supported");
+ }
+
+ /**
+ * to get the long value
+ *
+ * @param index
+ * @return long value based on index
+ */
+ @Override
+ public long getLong(int index) {
+ throw new UnsupportedOperationException("Operation not supported");
+ }
+
+ /**
+ * to get the double value
+ *
+ * @param index
+ * @return double value based on index
+ */
+ @Override
+ public double getDouble(int index) {
+ throw new UnsupportedOperationException("Operation not supported");
+ }
+
+ /**
+ * To get the bigdecimal value
+ *
+ * @param index
+ * @return bigdecimal value based on index
+ */
+ @Override
+ public BigDecimal getBigDecimal(int index) {
+ throw new UnsupportedOperationException("Operation not supported");
+ }
+
+ /**
+ * To free the occupied memory
+ */
+ @Override
+ public void freeMemory() {
+ // do nothing as GC will take care of freeing the memory
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeByteMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeByteMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeByteMeasureChunkStore.java
new file mode 100644
index 0000000..1f69cd2
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeByteMeasureChunkStore.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe;
+
+/**
+ * Responsible for storing Byte array data to memory.
+ */
+public class SafeByteMeasureChunkStore extends
+ SafeAbstractMeasureDataChunkStore<byte[]> {
+
+ /**
+ * data
+ */
+ private byte[] data;
+
+ public SafeByteMeasureChunkStore(int numberOfRows) {
+ super(numberOfRows);
+ }
+
+ /**
+ * Below method will be used to put byte array data to memory
+ *
+ * @param data
+ */
+ @Override
+ public void putData(byte[] data) {
+ this.data = data;
+ }
+
+ /**
+ * to get the byte value
+ *
+ * @param index
+ * @return byte value based on index
+ */
+ @Override
+ public byte getByte(int index) {
+ return this.data[index];
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeDoubleMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeDoubleMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeDoubleMeasureChunkStore.java
new file mode 100644
index 0000000..dc485c6
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeDoubleMeasureChunkStore.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe;
+
+/**
+ * Below class will be used to store the measure values of double data type
+ */
+public class SafeDoubleMeasureChunkStore extends
+ SafeAbstractMeasureDataChunkStore<double[]> {
+
+ /**
+ * data
+ */
+ private double[] data;
+
+ public SafeDoubleMeasureChunkStore(int numberOfRows) {
+ super(numberOfRows);
+ }
+
+ /**
+ * Below method will be used to store double array data
+ *
+ * @param data
+ */
+ @Override
+ public void putData(double[] data) {
+ this.data = data;
+ }
+
+ /**
+ * to get the double value
+ *
+ * @param index
+ * @return double value based on index
+ */
+ @Override
+ public double getDouble(int index) {
+ return this.data[index];
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java
new file mode 100644
index 0000000..3f5afc5
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeFixedLengthDimensionDataChunkStore.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe;
+
+import org.apache.carbondata.core.util.ByteUtil;
+
+/**
+ * Below class will be used to store fixed length dimension data
+ */
+public class SafeFixedLengthDimensionDataChunkStore extends SafeAbsractDimensionDataChunkStore {
+
+ /**
+ * Size of each value
+ */
+ private int columnValueSize;
+
+ public SafeFixedLengthDimensionDataChunkStore(boolean isInvertedIndex, int columnValueSize) {
+ super(isInvertedIndex);
+ this.columnValueSize = columnValueSize;
+ }
+
+ /**
+ * Below method will be used to get the row based inverted index
+ *
+ * @param rowId Inverted index
+ */
+ @Override public byte[] getRow(int rowId) {
+ // if column was explicitly sorted we need to get the rowid based inverted index reverse
+ if (isExplictSorted) {
+ rowId = invertedIndexReverse[rowId];
+ }
+ // creating a row
+ byte[] row = new byte[columnValueSize];
+ //copy the row from data chunk based on offset
+ // offset position will be index * each column value length
+ System.arraycopy(this.data, rowId * columnValueSize, row, 0, columnValueSize);
+ return row;
+ }
+
+ /**
+ * Below method will be used to get the surrogate key of the based on the row
+ * id passed
+ *
+ * @param rowId row id
+ * @return surrogate key
+ */
+ @Override public int getSurrogate(int index) {
+ // if column was explicitly sorted we need to get the rowid based inverted index reverse
+ if (isExplictSorted) {
+ index = invertedIndexReverse[index];
+ }
+ // below part is to convert the byte array to surrogate value
+ int startOffsetOfData = index * columnValueSize;
+ int surrogate = 0;
+ for (int i = 0; i < columnValueSize; i++) {
+ surrogate <<= 8;
+ surrogate ^= data[startOffsetOfData] & 0xFF;
+ startOffsetOfData++;
+ }
+ return surrogate;
+ }
+
+ /**
+ * Below method will be used to fill the row values to buffer array
+ *
+ * @param rowId row id of the data to be filled
+ * @param data buffer in which data will be filled
+ * @param offset off the of the buffer
+ */
+ @Override public void fillRow(int rowId, byte[] buffer, int offset) {
+ // if column was explicitly sorted we need to get the rowid based inverted index reverse
+ if (isExplictSorted) {
+ rowId = invertedIndexReverse[rowId];
+ }
+ //copy the row from memory block based on offset
+ // offset position will be index * each column value length
+ System.arraycopy(data, rowId * columnValueSize, buffer, offset, columnValueSize);
+ }
+
+ /**
+ * @return size of each column value
+ */
+ @Override public int getColumnValueSize() {
+ return columnValueSize;
+ }
+
+ /**
+ * to compare the two byte array
+ *
+ * @param index index of first byte array
+ * @param compareValue value of to be compared
+ * @return compare result
+ */
+ @Override public int compareTo(int index, byte[] compareValue) {
+ return ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(data, index * columnValueSize, columnValueSize, compareValue, 0,
+ columnValueSize);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeIntMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeIntMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeIntMeasureChunkStore.java
new file mode 100644
index 0000000..29b77cb
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeIntMeasureChunkStore.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe;
+
+/**
+ * Responsible for storing int array data to memory.
+ */
+public class SafeIntMeasureChunkStore extends
+ SafeAbstractMeasureDataChunkStore<int[]> {
+
+ /**
+ * data
+ */
+ private int[] data;
+
+ public SafeIntMeasureChunkStore(int numberOfRows) {
+ super(numberOfRows);
+ }
+
+ /**
+ * Below method will be used to put int array data to memory
+ *
+ * @param data
+ */
+ @Override
+ public void putData(int[] data) {
+ this.data = data;
+ }
+
+ /**
+ * to get the int value
+ *
+ * @param index
+ * @return int value based on index
+ */
+ @Override
+ public int getInt(int index) {
+ return this.data[index];
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeLongMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeLongMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeLongMeasureChunkStore.java
new file mode 100644
index 0000000..3fd0370
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeLongMeasureChunkStore.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe;
+
+/**
+ * Below class will be used to store the measure values of long data type
+ *
+ */
+public class SafeLongMeasureChunkStore extends
+ SafeAbstractMeasureDataChunkStore<long[]> {
+
+ /**
+ * data
+ */
+ private long[] data;
+
+ public SafeLongMeasureChunkStore(int numberOfRows) {
+ super(numberOfRows);
+ }
+
+ /**
+ * Below method will be used to store long array data
+ *
+ * @param data
+ */
+ @Override
+ public void putData(long[] data) {
+ this.data = data;
+ }
+
+ /**
+ * to get the long value
+ *
+ * @param index
+ * @return long value based on index
+ */
+ @Override
+ public long getLong(int index) {
+ return this.data[index];
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeShortMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeShortMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeShortMeasureChunkStore.java
new file mode 100644
index 0000000..269ad79
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeShortMeasureChunkStore.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe;
+
+/**
+ * Below class will be used to store the measure values of short data type
+ *
+ */
+public class SafeShortMeasureChunkStore extends
+ SafeAbstractMeasureDataChunkStore<short[]> {
+
+ /**
+ * data
+ */
+ private short[] data;
+
+ public SafeShortMeasureChunkStore(int numberOfRows) {
+ super(numberOfRows);
+ }
+
+ /**
+ * Below method will be used to put short array data
+ *
+ * @param data
+ */
+ @Override
+ public void putData(short[] data) {
+ this.data = data;
+ }
+
+ /**
+ * to get the short value
+ *
+ * @param index
+ * @return shot value based on index
+ */
+ @Override
+ public short getShort(int index) {
+ return data[index];
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
new file mode 100644
index 0000000..7d5ccd2
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/safe/SafeVariableLengthDimensionDataChunkStore.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.safe;
+
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.ByteUtil;
+
+/**
+ * Below class is responsible to store variable length dimension data chunk in
+ * memory Memory occupied can be on heap or offheap using unsafe interface
+ */
+public class SafeVariableLengthDimensionDataChunkStore extends SafeAbsractDimensionDataChunkStore {
+
+ /**
+ * total number of rows
+ */
+ private int numberOfRows;
+
+ /**
+ * offset of the data this will be used during search, as we can directly jump
+ * to particular location
+ */
+ private int[] dataOffsets;
+
+ public SafeVariableLengthDimensionDataChunkStore(boolean isInvertedIndex, int numberOfRows) {
+ super(isInvertedIndex);
+ this.numberOfRows = numberOfRows;
+ this.dataOffsets = new int[numberOfRows];
+ }
+
+ /**
+ * Below method will be used to put the rows and its metadata in offheap
+ *
+ * @param invertedIndex inverted index to be stored
+ * @param invertedIndexReverse inverted index reverse to be stored
+ * @param data data to be stored
+ */
+ @Override public void putArray(final int[] invertedIndex, final int[] invertedIndexReverse,
+ byte[] data) {
+ // first put the data, inverted index and reverse inverted index to memory
+ super.putArray(invertedIndex, invertedIndexReverse, data);
+ // As data is of variable length and data format is
+ // <length in short><data><length in short><data>
+ // we need to store offset of each data so data can be accessed directly
+ // for example:
+ //data = {0,5,1,2,3,4,5,0,6,0,1,2,3,4,5,0,2,8,9}
+ //so value stored in offset will be position of actual data
+ // [2,9,17]
+ // to store this value we need to get the actual data length + 2 bytes used for storing the
+ // length
+
+ // start position will be used to store the current data position
+ int startOffset = 0;
+ // as first position will be start from 2 byte as data is stored first in the memory block
+ // we need to skip first two bytes this is because first two bytes will be length of the data
+ // which we have to skip
+ dataOffsets[0] = CarbonCommonConstants.SHORT_SIZE_IN_BYTE;
+ // creating a byte buffer which will wrap the length of the row
+ ByteBuffer buffer = ByteBuffer.allocate(CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+ for (int i = 1; i < numberOfRows; i++) {
+ buffer.put(data, startOffset, CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+ buffer.flip();
+ // so current row position will be
+ // previous row length + 2 bytes used for storing previous row data
+ startOffset += buffer.getShort() + CarbonCommonConstants.SHORT_SIZE_IN_BYTE;
+ // as same byte buffer is used to avoid creating many byte buffer for each row
+ // we need to clear the byte buffer
+ buffer.clear();
+ dataOffsets[i] = startOffset + CarbonCommonConstants.SHORT_SIZE_IN_BYTE;
+ }
+ }
+
+ @Override public byte[] getRow(int rowId) {
+ // if column was explicitly sorted we need to get the rowid based inverted index reverse
+ if (isExplictSorted) {
+ rowId = invertedIndexReverse[rowId];
+ }
+ // now to get the row from memory block we need to do following thing
+ // 1. first get the current offset
+ // 2. if it's not a last row- get the next row offset
+ // Subtract the current row offset + 2 bytes(to skip the data length) with next row offset
+ // else subtract the current row offset with complete data
+ // length get the offset of set of data
+ int currentDataOffset = dataOffsets[rowId];
+ short length = 0;
+ // calculating the length of data
+ if (rowId < numberOfRows - 1) {
+ length = (short) (dataOffsets[rowId + 1] - (currentDataOffset
+ + CarbonCommonConstants.SHORT_SIZE_IN_BYTE));
+ } else {
+ // for last record
+ length = (short) (this.data.length - currentDataOffset);
+ }
+ byte[] currentRowData = new byte[length];
+ System.arraycopy(data, currentDataOffset, currentRowData, 0, length);
+ return currentRowData;
+ }
+
+ @Override public int compareTo(int index, byte[] compareValue) {
+ // now to get the row from memory block we need to do following thing
+ // 1. first get the current offset
+ // 2. if it's not a last row- get the next row offset
+ // Subtract the current row offset + 2 bytes(to skip the data length) with next row offset
+ // else subtract the current row offset with complete data
+ // length
+
+ // get the offset of set of data
+ int currentDataOffset = dataOffsets[index];
+ short length = 0;
+ // calculating the length of data
+ if (index < numberOfRows - 1) {
+ length = (short) (dataOffsets[index + 1] - (currentDataOffset
+ + CarbonCommonConstants.SHORT_SIZE_IN_BYTE));
+ } else {
+ // for last record
+ length = (short) (this.data.length - currentDataOffset);
+ }
+ return ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(data, currentDataOffset, length, compareValue, 0, compareValue.length);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
new file mode 100644
index 0000000..044e5f4
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.store.DimensionDataChunkStore;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
+import org.apache.carbondata.core.memory.MemoryBlock;
+import org.apache.carbondata.core.unsafe.CarbonUnsafe;
+
+/**
+ * Responsibility is to store dimension data in memory. storage can be on heap
+ * or offheap.
+ */
+public abstract class UnsafeAbstractDimensionDataChunkStore implements DimensionDataChunkStore {
+
+ /**
+ * memory block for data page
+ */
+ protected MemoryBlock dataPageMemoryBlock;
+
+ /**
+ * to check whether dimension column was explicitly sorted or not
+ */
+ protected boolean isExplicitSorted;
+
+ /**
+ * is memory released
+ */
+ protected boolean isMemoryReleased;
+
+ /**
+ * length of the actual data
+ */
+ protected int dataLength;
+
+ /**
+ * offset of the inverted index reverse
+ */
+ protected long invertedIndexReverseOffset;
+
+ /**
+ * to validate whether data is already kept in memory or not
+ */
+ protected boolean isMemoryOccupied;
+
+ /**
+ * Constructor
+ *
+ * @param totalSize total size of the data to be kept
+ * @param isInvertedIdex is inverted index present
+ * @param numberOfRows total number of rows
+ */
+ public UnsafeAbstractDimensionDataChunkStore(long totalSize, boolean isInvertedIdex,
+ int numberOfRows) {
+ // allocating the data page
+ this.dataPageMemoryBlock =
+ MemoryAllocatorFactory.INSATANCE.getMemoryAllocator().allocate(totalSize);
+ this.isExplicitSorted = isInvertedIdex;
+ }
+
+ /**
+ * Below method will be used to put the rows and its metadata in offheap
+ *
+ * @param invertedIndex inverted index to be stored
+ * @param invertedIndexReverse inverted index reverse to be stored
+ * @param data data to be stored
+ */
+ @Override public void putArray(final int[] invertedIndex, final int[] invertedIndexReverse,
+ final byte[] data) {
+ assert (!isMemoryOccupied);
+ this.dataLength = data.length;
+ this.invertedIndexReverseOffset = dataLength;
+ if (isExplicitSorted) {
+ this.invertedIndexReverseOffset +=
+ invertedIndex.length * CarbonCommonConstants.INT_SIZE_IN_BYTE;
+ }
+ // copy the data to memory
+ CarbonUnsafe.unsafe
+ .copyMemory(data, CarbonUnsafe.BYTE_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset(), this.dataLength);
+ // if inverted index is present then copy the inverted index
+ // and reverse inverted index to memory
+ if (isExplicitSorted) {
+ CarbonUnsafe.unsafe.copyMemory(invertedIndex, CarbonUnsafe.INT_ARRAY_OFFSET,
+ dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + dataLength,
+ invertedIndex.length * CarbonCommonConstants.INT_SIZE_IN_BYTE);
+ CarbonUnsafe.unsafe.copyMemory(invertedIndexReverse, CarbonUnsafe.INT_ARRAY_OFFSET,
+ dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset,
+ invertedIndexReverse.length * CarbonCommonConstants.INT_SIZE_IN_BYTE);
+ }
+ }
+
+ /**
+ * Below method will be used to free the memory occupied by the column chunk
+ */
+ @Override public void freeMemory() {
+ if (isMemoryReleased) {
+ return;
+ }
+ // free data page memory
+ MemoryAllocatorFactory.INSATANCE.getMemoryAllocator().free(dataPageMemoryBlock);
+ isMemoryReleased = true;
+ this.dataPageMemoryBlock = null;
+ this.isMemoryOccupied = false;
+ }
+
+ /**
+ * Below method will be used to get the inverted index
+ *
+ * @param rowId row id
+ * @return inverted index based on row id passed
+ */
+ @Override public int getInvertedIndex(int rowId) {
+ return CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + dataLength + (rowId
+ * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+ }
+
+ /**
+ * Below method will be used to get the surrogate key of the based on the row
+ * id passed
+ *
+ * @param rowId row id
+ * @return surrogate key
+ */
+ @Override public int getSurrogate(int rowId) {
+ throw new UnsupportedOperationException("Operation not supported");
+ }
+
+ /**
+ * @return size of each column value
+ */
+ @Override public int getColumnValueSize() {
+ throw new UnsupportedOperationException("Operation not supported");
+ }
+
+ /**
+ * @return whether column was explicitly sorted or not
+ */
+ @Override public boolean isExplicitSorted() {
+ return isExplicitSorted;
+ }
+
+ /**
+ * Below method will be used to fill the row values to data array
+ *
+ * @param rowId row id of the data to be filled
+ * @param data buffer in which data will be filled
+ * @param offset off the of the buffer
+ */
+ @Override public void fillRow(int rowId, byte[] data, int offset) {
+ throw new UnsupportedOperationException("Operation not supported");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractMeasureDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractMeasureDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractMeasureDataChunkStore.java
new file mode 100644
index 0000000..98cb2f6
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeAbstractMeasureDataChunkStore.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe;
+
+import java.math.BigDecimal;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.store.MeasureDataChunkStore;
+import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
+import org.apache.carbondata.core.memory.MemoryBlock;
+
+/**
+ * Responsibility is store the measure data in memory, memory can be on heap or
+ * offheap based on the user configuration using unsafe interface
+ */
+public abstract class UnsafeAbstractMeasureDataChunkStore<T> implements MeasureDataChunkStore<T> {
+
+ /**
+ * memory block
+ */
+ protected MemoryBlock dataPageMemoryBlock;
+
+ /**
+ * number of rows
+ */
+ protected int numberOfRows;
+
+ /**
+ * to check memory is released or not
+ */
+ protected boolean isMemoryReleased;
+
+ /**
+ * to check memory is occupied or not
+ */
+ protected boolean isMemoryOccupied;
+
+ public UnsafeAbstractMeasureDataChunkStore(int numberOfRows) {
+ this.numberOfRows = numberOfRows;
+ }
+
+ /**
+ * to get the byte value
+ *
+ * @param index
+ * @return byte value based on index
+ */
+ @Override public byte getByte(int index) {
+ throw new UnsupportedOperationException("Operation not supported");
+ }
+
+ /**
+ * to get the short value
+ *
+ * @param index
+ * @return short value based on index
+ */
+ @Override public short getShort(int index) {
+ throw new UnsupportedOperationException("Operation not supported");
+ }
+
+ /**
+ * to get the int value
+ *
+ * @param index
+ * @return int value based on index
+ */
+ @Override public int getInt(int index) {
+ throw new UnsupportedOperationException("Operation not supported");
+ }
+
+ /**
+ * to get the long value
+ *
+ * @param index
+ * @return long value based on index
+ */
+ @Override public long getLong(int index) {
+ throw new UnsupportedOperationException("Operation not supported");
+ }
+
+ /**
+ * to get the double value
+ *
+ * @param index
+ * @return double value based on index
+ */
+ @Override public double getDouble(int index) {
+ throw new UnsupportedOperationException("Operation not supported");
+ }
+
+ /**
+ * To get the bigdecimal value
+ *
+ * @param index
+ * @return bigdecimal value based on index
+ */
+ @Override public BigDecimal getBigDecimal(int index) {
+ throw new UnsupportedOperationException("Operation not supported");
+ }
+
+ /**
+ * To free the occupied memory
+ */
+ @Override public void freeMemory() {
+ if (isMemoryReleased) {
+ return;
+ }
+ MemoryAllocatorFactory.INSATANCE.getMemoryAllocator().free(dataPageMemoryBlock);
+ isMemoryReleased = true;
+ this.dataPageMemoryBlock = null;
+ this.isMemoryOccupied = false;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java
new file mode 100644
index 0000000..5d0a802
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeByteMeasureChunkStore.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe;
+
+import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
+import org.apache.carbondata.core.unsafe.CarbonUnsafe;
+
+/**
+ * Responsible for storing Byte array data to memory. memory can be on heap or
+ * offheap based on the user configuration using unsafe interface
+ */
+public class UnsafeByteMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<byte[]> {
+
+ public UnsafeByteMeasureChunkStore(int numberOfRows) {
+ super(numberOfRows);
+ }
+
+ /**
+ * Below method will be used to put byte array data to memory
+ *
+ * @param data
+ */
+ @Override public void putData(byte[] data) {
+ assert (!this.isMemoryOccupied);
+ this.dataPageMemoryBlock =
+ MemoryAllocatorFactory.INSATANCE.getMemoryAllocator().allocate(data.length);
+ // copy the data to memory
+ CarbonUnsafe.unsafe
+ .copyMemory(data, CarbonUnsafe.BYTE_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size());
+ this.isMemoryOccupied = true;
+ }
+
+ /**
+ * to get the byte value
+ *
+ * @param index
+ * @return byte value based on index
+ */
+ @Override public byte getByte(int index) {
+ return CarbonUnsafe.unsafe
+ .getByte(dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + index);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java
new file mode 100644
index 0000000..9c61ff2
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeDoubleMeasureChunkStore.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
+import org.apache.carbondata.core.unsafe.CarbonUnsafe;
+
+/**
+ * Responsible for storing double array data to memory. memory can be on heap or
+ * offheap based on the user configuration using unsafe interface
+ */
+public class UnsafeDoubleMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<double[]> {
+
+ public UnsafeDoubleMeasureChunkStore(int numberOfRows) {
+ super(numberOfRows);
+ }
+
+ /**
+ * Below method will be used to put double array data to memory
+ *
+ * @param data
+ */
+ @Override public void putData(double[] data) {
+ assert (!this.isMemoryOccupied);
+ this.dataPageMemoryBlock = MemoryAllocatorFactory.INSATANCE.getMemoryAllocator()
+ .allocate(data.length * CarbonCommonConstants.DOUBLE_SIZE_IN_BYTE);
+ // copy the data to memory
+ CarbonUnsafe.unsafe
+ .copyMemory(data, CarbonUnsafe.DOUBLE_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size());
+ this.isMemoryOccupied = true;
+ }
+
+ /**
+ * to get the double value
+ *
+ * @param index
+ * @return double value based on index
+ */
+ @Override public double getDouble(int index) {
+ return CarbonUnsafe.unsafe.getDouble(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + (index * CarbonCommonConstants.DOUBLE_SIZE_IN_BYTE));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
new file mode 100644
index 0000000..ff28559
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.unsafe.CarbonUnsafe;
+
+/**
+ * Below class is responsible to store fixed length dimension data chunk in
+ * memory Memory occupied can be on heap or offheap using unsafe interface
+ */
+public class UnsafeFixedLengthDimensionDataChunkStore
+ extends UnsafeAbstractDimensionDataChunkStore {
+
+ /**
+ * Size of each value
+ */
+ private int columnValueSize;
+
+ /**
+ * Constructor
+ *
+ * @param columnValueSize value of each column
+ * @param isInvertedIdex is inverted index present
+ * @param numberOfRows total number of rows
+ */
+ public UnsafeFixedLengthDimensionDataChunkStore(long totalDataSize, int columnValueSize,
+ boolean isInvertedIdex, int numberOfRows) {
+ super(totalDataSize, isInvertedIdex, numberOfRows);
+ this.columnValueSize = columnValueSize;
+ }
+
+ /**
+ * Below method will be used to get the row based inverted index
+ *
+ * @param rowId Inverted index
+ */
+ @Override public byte[] getRow(int rowId) {
+ // if column was explicitly sorted we need to get the rowid based inverted index reverse
+ if (isExplicitSorted) {
+ rowId = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (rowId
+ * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+ }
+ // creating a row
+ byte[] data = new byte[columnValueSize];
+ //copy the row from memory block based on offset
+ // offset position will be index * each column value length
+ CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + (rowId * columnValueSize), data,
+ CarbonUnsafe.BYTE_ARRAY_OFFSET, columnValueSize);
+ return data;
+ }
+
+ /**
+ * Below method will be used to get the surrogate key of the based on the row
+ * id passed
+ *
+ * @param rowId row id
+ * @return surrogate key
+ */
+ @Override public int getSurrogate(int index) {
+ // if column was explicitly sorted we need to get the rowid based inverted index reverse
+ if (isExplicitSorted) {
+ index = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (index
+ * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+ }
+ // below part is to convert the byte array to surrogate value
+ int startOffsetOfData = index * columnValueSize;
+ int surrogate = 0;
+ for (int i = 0; i < columnValueSize; i++) {
+ surrogate <<= 8;
+ surrogate ^= CarbonUnsafe.unsafe.getByte(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + startOffsetOfData) & 0xFF;
+ startOffsetOfData++;
+ }
+ return surrogate;
+ }
+
+ /**
+ * Below method will be used to fill the row values to buffer array
+ *
+ * @param rowId row id of the data to be filled
+ * @param data buffer in which data will be filled
+ * @param offset off the of the buffer
+ */
+ @Override public void fillRow(int rowId, byte[] buffer, int offset) {
+ // if column was explicitly sorted we need to get the rowid based inverted index reverse
+ if (isExplicitSorted) {
+ rowId = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (rowId
+ * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+ }
+ //copy the row from memory block based on offset
+ // offset position will be index * each column value length
+ CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + (rowId * columnValueSize), buffer,
+ CarbonUnsafe.BYTE_ARRAY_OFFSET + offset, columnValueSize);
+ }
+
+ /**
+ * @return size of each column value
+ */
+ @Override public int getColumnValueSize() {
+ return columnValueSize;
+ }
+
+ /**
+ * to compare the two byte array
+ *
+ * @param index index of first byte array
+ * @param compareValue value of to be compared
+ * @return compare result
+ */
+ @Override public int compareTo(int index, byte[] compareValue) {
+ // based on index we need to calculate the actual position in memory block
+ index = index * columnValueSize;
+ int compareResult = 0;
+ for (int i = 0; i < compareValue.length; i++) {
+ compareResult = (CarbonUnsafe.unsafe
+ .getByte(dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + index)
+ & 0xff) - (compareValue[i] & 0xff);
+ if (compareResult != 0) {
+ break;
+ }
+ index++;
+ }
+ return compareResult;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeIntMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeIntMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeIntMeasureChunkStore.java
new file mode 100644
index 0000000..0111648
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeIntMeasureChunkStore.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
+import org.apache.carbondata.core.unsafe.CarbonUnsafe;
+
+/**
+ * Responsible for storing int array data to memory. memory can be on heap or
+ * offheap based on the user configuration using unsafe interface
+ */
+public class UnsafeIntMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<int[]> {
+
+ public UnsafeIntMeasureChunkStore(int numberOfRows) {
+ super(numberOfRows);
+ }
+
+ /**
+ * Below method will be used to put int array data to memory
+ *
+ * @param data
+ */
+ @Override public void putData(int[] data) {
+ assert (!this.isMemoryOccupied);
+ this.dataPageMemoryBlock = MemoryAllocatorFactory.INSATANCE.getMemoryAllocator()
+ .allocate(data.length * CarbonCommonConstants.INT_SIZE_IN_BYTE);
+ // copy the data to memory
+ CarbonUnsafe.unsafe
+ .copyMemory(data, CarbonUnsafe.INT_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size());
+ this.isMemoryOccupied = true;
+ }
+
+ /**
+ * to get the int value
+ *
+ * @param index
+ * @return int value based on index
+ */
+ @Override public int getInt(int index) {
+ return CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + (index * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeLongMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeLongMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeLongMeasureChunkStore.java
new file mode 100644
index 0000000..d3bed25
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeLongMeasureChunkStore.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
+import org.apache.carbondata.core.unsafe.CarbonUnsafe;
+
+/**
+ * Responsible for storing long array data to memory. memory can be on heap or
+ * offheap based on the user configuration using unsafe interface
+ */
+public class UnsafeLongMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<long[]> {
+
+ public UnsafeLongMeasureChunkStore(int numberOfRows) {
+ super(numberOfRows);
+ }
+
+ /**
+ * Below method will be used to put long array data to memory
+ *
+ * @param data
+ */
+ @Override public void putData(long[] data) {
+ assert (!this.isMemoryOccupied);
+ this.dataPageMemoryBlock = MemoryAllocatorFactory.INSATANCE.getMemoryAllocator()
+ .allocate(data.length * CarbonCommonConstants.LONG_SIZE_IN_BYTE);
+ // copy the data to memory
+ CarbonUnsafe.unsafe
+ .copyMemory(data, CarbonUnsafe.LONG_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size());
+ this.isMemoryOccupied = true;
+ }
+
+ /**
+ * to get the long value
+ *
+ * @param index
+ * @return long value based on index
+ */
+ @Override public long getLong(int index) {
+ return CarbonUnsafe.unsafe.getLong(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + (index * CarbonCommonConstants.LONG_SIZE_IN_BYTE));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeShortMeasureChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeShortMeasureChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeShortMeasureChunkStore.java
new file mode 100644
index 0000000..ba865cd
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeShortMeasureChunkStore.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.memory.MemoryAllocatorFactory;
+import org.apache.carbondata.core.unsafe.CarbonUnsafe;
+
+/**
+ * Responsible for storing short array data to memory. memory can be on heap or
+ * offheap based on the user configuration using unsafe interface
+ */
+public class UnsafeShortMeasureChunkStore extends UnsafeAbstractMeasureDataChunkStore<short[]> {
+
+ public UnsafeShortMeasureChunkStore(int numberOfRows) {
+ super(numberOfRows);
+ }
+
+ /**
+ * Below method will be used to put short array data to memory
+ *
+ * @param data
+ */
+ @Override public void putData(short[] data) {
+ assert (!this.isMemoryOccupied);
+ this.dataPageMemoryBlock = MemoryAllocatorFactory.INSATANCE.getMemoryAllocator()
+ .allocate(data.length * CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+ // copy the data to memory
+ CarbonUnsafe.unsafe
+ .copyMemory(data, CarbonUnsafe.SHORT_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset(), dataPageMemoryBlock.size());
+ this.isMemoryOccupied = true;
+ }
+
+ /**
+ * to get the short value
+ *
+ * @param index
+ * @return shot value based on index
+ */
+ @Override public short getShort(int index) {
+ return CarbonUnsafe.unsafe.getShort(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + (index * CarbonCommonConstants.SHORT_SIZE_IN_BYTE));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
new file mode 100644
index 0000000..c6155e7
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.store.impl.unsafe;
+
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.unsafe.CarbonUnsafe;
+
+/**
+ * Below class is responsible to store variable length dimension data chunk in
+ * memory Memory occupied can be on heap or offheap using unsafe interface
+ */
+public class UnsafeVariableLengthDimesionDataChunkStore
+ extends UnsafeAbstractDimensionDataChunkStore {
+
+ /**
+ * total number of rows
+ */
+ private int numberOfRows;
+
+ /**
+ * pointers offsets
+ */
+ private long dataPointersOffsets;
+
+ public UnsafeVariableLengthDimesionDataChunkStore(long totalSize, boolean isInvertedIdex,
+ int numberOfRows) {
+ super(totalSize, isInvertedIdex, numberOfRows);
+ this.numberOfRows = numberOfRows;
+ }
+
+ /**
+ * Below method will be used to put the rows and its metadata in offheap
+ *
+ * @param invertedIndex inverted index to be stored
+ * @param invertedIndexReverse inverted index reverse to be stored
+ * @param data data to be stored
+ */
+ @Override public void putArray(final int[] invertedIndex, final int[] invertedIndexReverse,
+ byte[] data) {
+ // first put the data, inverted index and reverse inverted index to memory
+ super.putArray(invertedIndex, invertedIndexReverse, data);
+ // position from where offsets will start
+ this.dataPointersOffsets = this.invertedIndexReverseOffset;
+ if (isExplicitSorted) {
+ this.dataPointersOffsets += numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE;
+ }
+ // As data is of variable length and data format is
+ // <length in short><data><length in short><data>
+ // we need to store offset of each data so data can be accessed directly
+ // for example:
+ //data = {0,5,1,2,3,4,5,0,6,0,1,2,3,4,5,0,2,8,9}
+ //so value stored in offset will be position of actual data
+ // [2,9,17]
+ // to store this value we need to get the actual data length + 2 bytes used for storing the
+ // length
+
+ // start position will be used to store the current data position
+ int startOffset = 0;
+ // position from where offsets will start
+ long pointerOffsets = this.dataPointersOffsets;
+ // as first position will be start from 2 byte as data is stored first in the memory block
+ // we need to skip first two bytes this is because first two bytes will be length of the data
+ // which we have to skip
+ CarbonUnsafe.unsafe.putInt(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + pointerOffsets,
+ CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+ // incrementing the pointers as first value is already filled and as we are storing as int
+ // we need to increment the 4 bytes to set the position of the next value to set
+ pointerOffsets += CarbonCommonConstants.INT_SIZE_IN_BYTE;
+ // creating a byte buffer which will wrap the length of the row
+ // using byte buffer as unsafe will return bytes in little-endian encoding
+ ByteBuffer buffer = ByteBuffer.allocate(CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+ // store length of data
+ byte[] length = new byte[CarbonCommonConstants.SHORT_SIZE_IN_BYTE];
+ // as first offset is already stored, we need to start from the 2nd row in data array
+ for (int i = 1; i < numberOfRows; i++) {
+ // first copy the length of previous row
+ CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + startOffset, length, CarbonUnsafe.BYTE_ARRAY_OFFSET,
+ CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+ buffer.put(length);
+ buffer.flip();
+ // so current row position will be
+ // previous row length + 2 bytes used for storing previous row data
+ startOffset += CarbonCommonConstants.SHORT_SIZE_IN_BYTE + buffer.getShort();
+ // as same byte buffer is used to avoid creating many byte buffer for each row
+ // we need to clear the byte buffer
+ buffer.clear();
+ // now put the offset of current row, here we need to add 2 more bytes as current will
+ // also have length part so we have to skip length
+ CarbonUnsafe.unsafe.putInt(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + pointerOffsets,
+ startOffset + CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
+ // incrementing the pointers as first value is already filled and as we are storing as int
+ // we need to increment the 4 bytes to set the position of the next value to set
+ pointerOffsets += CarbonCommonConstants.INT_SIZE_IN_BYTE;
+ }
+
+ }
+
+ /**
+ * Below method will be used to get the row based on row id passed
+ *
+ * @param index
+ * @return row
+ */
+ @Override public byte[] getRow(int rowId) {
+ byte[] data = null;
+ try {
+ // if column was explicitly sorted we need to get the rowid based inverted index reverse
+ if (isExplicitSorted) {
+ rowId = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (rowId
+ * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+ }
+ // now to get the row from memory block we need to do following thing
+ // 1. first get the current offset
+ // 2. if it's not a last row- get the next row offset
+ // Subtract the current row offset + 2 bytes(to skip the data length) with next row offset
+ // else subtract the current row offset + 2 bytes(to skip the data length)
+ // with complete data length
+ int currentDataOffset = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + (rowId
+ * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+ short length = 0;
+ // calculating the length of data
+ if (rowId < numberOfRows - 1) {
+ int OffsetOfNextdata = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((rowId + 1)
+ * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+ length = (short) (OffsetOfNextdata - (currentDataOffset
+ + CarbonCommonConstants.SHORT_SIZE_IN_BYTE));
+ } else {
+ // for last record we need to subtract with data length
+ length = (short) (this.dataLength - currentDataOffset);
+ }
+ data = new byte[length];
+ CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + currentDataOffset, data,
+ CarbonUnsafe.BYTE_ARRAY_OFFSET, length);
+ } catch (Throwable t) {
+ System.out.println();
+ }
+ return data;
+ }
+
+ /**
+ * to compare the two byte array
+ *
+ * @param index index of first byte array
+ * @param compareValue value of to be compared
+ * @return compare result
+ */
+ @Override public int compareTo(int index, byte[] compareValue) {
+ // now to get the row from memory block we need to do following thing
+ // 1. first get the current offset
+ // 2. if it's not a last row- get the next row offset
+ // Subtract the current row offset + 2 bytes(to skip the data length) with next row offset
+ // else subtract the current row offset
+ // with complete data length get the offset of set of data
+ int currentDataOffset = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + (index
+ * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+ short length = 0;
+ // calculating the length of data
+ if (index < numberOfRows - 1) {
+ int OffsetOfNextdata = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((index + 1)
+ * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+ length = (short) (OffsetOfNextdata - (currentDataOffset
+ + CarbonCommonConstants.SHORT_SIZE_IN_BYTE));
+ } else {
+ // for last record we need to subtract with data length
+ length = (short) (this.dataLength - currentDataOffset);
+ }
+ // as this class handles this variable length data, so filter value can be
+ // smaller or bigger than than actual data, so we need to take the smaller length
+ int compareResult = 0;
+ int compareLength = length < compareValue.length ? length : compareValue.length;
+ for (int i = 0; i < compareLength; i++) {
+ compareResult = (CarbonUnsafe.unsafe.getByte(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + currentDataOffset) & 0xff) - (compareValue[i]
+ & 0xff);
+ // if compare result is not equal we can break
+ if (compareResult != 0) {
+ break;
+ }
+ // increment the offset by one as comparison is done byte by byte
+ currentDataOffset++;
+ }
+ return compareResult;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index f91a75c..041f5ed 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1081,6 +1081,7 @@ public final class CarbonCommonConstants {
/**
* Default carbon dictionary server port
+
*/
public static final String DICTIONARY_SERVER_PORT_DEFAULT = "2030";
@@ -1094,6 +1095,26 @@ public final class CarbonCommonConstants {
*/
public static final int DEFAULT_COLUMN_LENGTH = 100000;
+ /**
+ * property for enabling unsafe based query processing
+ */
+ public static final String ENABLE_UNSAFE_IN_QUERY_EXECUTION = "enable.unsafe.in.query.processing";
+
+ /**
+ * default property of unsafe processing
+ */
+ public static final String ENABLE_UNSAFE_IN_QUERY_EXECUTION_DEFAULTVALUE = "false";
+
+ /**
+ * property for offheap based processing
+ */
+ public static final String USE_OFFHEAP_IN_QUERY_PROCSSING = "use.offheap.in.query.processing";
+
+ /**
+ * default value of offheap based processing
+ */
+ public static final String USE_OFFHEAP_IN_QUERY_PROCSSING_DEFAULT = "true";
+
private CarbonCommonConstants() {
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java
index a174a18..e40dadd 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java
@@ -31,5 +31,4 @@ public interface NodeMeasureDataStore {
* @return writable array (compressed or normal)
*/
byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] dataHolderArray);
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/Compressor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/Compressor.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/Compressor.java
index a3d922d..1b90bc6 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/Compressor.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/Compressor.java
@@ -25,24 +25,36 @@ public interface Compressor {
byte[] unCompressByte(byte[] compInput);
+ byte[] unCompressByte(byte[] compInput, int offset, int length);
+
byte[] compressShort(short[] unCompInput);
short[] unCompressShort(byte[] compInput);
+ short[] unCompressShort(byte[] compInput, int offset, int lenght);
+
byte[] compressInt(int[] unCompInput);
int[] unCompressInt(byte[] compInput);
+ int[] unCompressInt(byte[] compInput, int offset, int length);
+
byte[] compressLong(long[] unCompInput);
long[] unCompressLong(byte[] compInput);
+ long[] unCompressLong(byte[] compInput, int offset, int length);
+
byte[] compressFloat(float[] unCompInput);
float[] unCompressFloat(byte[] compInput);
+ float[] unCompressFloat(byte[] compInput, int offset, int length);
+
byte[] compressDouble(double[] unCompInput);
double[] unCompressDouble(byte[] compInput);
+ double[] unCompressDouble(byte[] compInput, int offset, int length);
+
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8d9babe3/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/CompressorFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/CompressorFactory.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/CompressorFactory.java
index 110010f..5160879 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/CompressorFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/compression/CompressorFactory.java
@@ -24,19 +24,29 @@ import org.apache.carbondata.core.util.CarbonProperties;
public class CompressorFactory {
- private static Compressor snappy;
+ private static final CompressorFactory COMPRESSOR_FACTORY = new CompressorFactory();
- public static Compressor getInstance() {
- String compressor = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.COMPRESSOR,
- CarbonCommonConstants.DEFAULT_COMPRESSOR);
- switch (compressor) {
+ private final Compressor compressor;
+
+ private CompressorFactory() {
+ String compressorType = CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.COMPRESSOR, CarbonCommonConstants.DEFAULT_COMPRESSOR);
+ switch (compressorType) {
case "snappy":
- if (snappy == null) {
- snappy = new SnappyCompressor();
- }
- return snappy;
+ compressor = new SnappyCompressor();
+ break;
default:
- throw new RuntimeException("Unsupported compressor: " + compressor);
+ throw new RuntimeException(
+ "Invalid compressor type provided! Please provide valid compressor type");
}
}
+
+ public static CompressorFactory getInstance() {
+ return COMPRESSOR_FACTORY;
+ }
+
+ public Compressor getCompressor() {
+ return compressor;
+ }
+
}