You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2016/08/15 07:09:31 UTC
[46/52] [partial] incubator-carbondata git commit: Renamed packages
to org.apache.carbondata and fixed errors
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/TableTaskInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/TableTaskInfo.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/TableTaskInfo.java
new file mode 100644
index 0000000..1f8caf0
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/TableTaskInfo.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.block;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+/**
+ * This class is responsible for maintaining the mapping of tasks of a node.
+ */
+public class TableTaskInfo extends Distributable {
+
+ private final List<TableBlockInfo> tableBlockInfoList;
+ private final String taskId;
+ public String getTaskId() {
+ return taskId;
+ }
+
+ public List<TableBlockInfo> getTableBlockInfoList() {
+ return tableBlockInfoList;
+ }
+
+ public TableTaskInfo(String taskId, List<TableBlockInfo> tableBlockInfoList){
+ this.taskId = taskId;
+ this.tableBlockInfoList = tableBlockInfoList;
+ }
+
+ @Override public String[] getLocations() {
+ Set<String> locations = new HashSet<String>();
+ for(TableBlockInfo tableBlockInfo: tableBlockInfoList){
+ locations.addAll(Arrays.asList(tableBlockInfo.getLocations()));
+ }
+ locations.toArray(new String[locations.size()]);
+ List<String> nodes = TableTaskInfo.maxNoNodes(tableBlockInfoList);
+ return nodes.toArray(new String[nodes.size()]);
+ }
+
+ @Override public int compareTo(Distributable o) {
+ return taskId.compareTo(((TableTaskInfo)o).getTaskId());
+ }
+
+ /**
+ * Finding which node has the maximum number of blocks for it.
+ * @param blockList
+ * @return
+ */
+ public static List<String> maxNoNodes(List<TableBlockInfo> blockList) {
+ boolean useIndex = true;
+ Integer maxOccurence = 0;
+ String maxNode = null;
+ Map<String, Integer> nodeAndOccurenceMapping = new TreeMap<>();
+
+ // populate the map of node and number of occurences of that node.
+ for (TableBlockInfo block : blockList) {
+ for (String node : block.getLocations()) {
+ Integer nodeOccurence = nodeAndOccurenceMapping.get(node);
+ if (null == nodeOccurence) {
+ nodeAndOccurenceMapping.put(node, 1);
+ } else {
+ nodeOccurence++;
+ }
+ }
+ }
+ Integer previousValueOccurence = null;
+
+ // check which node is occured maximum times.
+ for (Map.Entry<String, Integer> entry : nodeAndOccurenceMapping.entrySet()) {
+ // finding the maximum node.
+ if (entry.getValue() > maxOccurence) {
+ maxOccurence = entry.getValue();
+ maxNode = entry.getKey();
+ }
+ // first time scenario. initialzing the previous value.
+ if (null == previousValueOccurence) {
+ previousValueOccurence = entry.getValue();
+ } else {
+ // for the case where all the nodes have same number of blocks then
+ // we need to return complete list instead of max node.
+ if (previousValueOccurence != entry.getValue()) {
+ useIndex = false;
+ }
+ }
+ }
+
+ // if all the nodes have equal occurence then returning the complete key set.
+ if (useIndex) {
+ return new ArrayList<>(nodeAndOccurenceMapping.keySet());
+ }
+
+ // if any max node is found then returning the max node.
+ List<String> node = new ArrayList<>(1);
+ node.add(maxNode);
+ return node;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/TaskBlockInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/TaskBlockInfo.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/TaskBlockInfo.java
new file mode 100644
index 0000000..1221cc1
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/TaskBlockInfo.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.carbon.datastore.block;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+
+/**
+ * This class contains blocks info of each task
+ */
+public class TaskBlockInfo {
+
+ // stores TableBlockInfo list of each task
+ private Map<String, List<TableBlockInfo>> taskBlockInfoMapping;
+
+ public TaskBlockInfo(){
+
+ taskBlockInfoMapping = new HashMap<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ }
+
+ /**
+ * returns task set
+ * @return
+ */
+ public Set<String> getTaskSet() {
+ return taskBlockInfoMapping.keySet();
+ }
+
+
+ /**
+ * returns TableBlockInfoList of given task
+ * @return
+ */
+ public List<TableBlockInfo> getTableBlockInfoList(String task) {
+ return taskBlockInfoMapping.get(task);
+ }
+
+ /**
+ * maps TableBlockInfoList to respective task
+ * @param task
+ * @param tableBlockInfoList
+ */
+ public void addTableBlockInfoList(String task, List<TableBlockInfo> tableBlockInfoList) {
+ taskBlockInfoMapping.put(task, tableBlockInfoList);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/DimensionChunkAttributes.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/DimensionChunkAttributes.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/DimensionChunkAttributes.java
new file mode 100644
index 0000000..4dcf083
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/DimensionChunkAttributes.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk;
+
+/**
+ * Dimension chunk attributes which holds all the
+ * property about the dimension chunk data
+ */
+public class DimensionChunkAttributes {
+
+ /**
+ * inverted index of the data
+ */
+ private int[] invertedIndexes;
+
+ /**
+ * reverse index of the data
+ */
+ private int[] invertedIndexesReverse;
+
+ /**
+ * each row size
+ */
+ private int columnValueSize;
+
+ /**
+ * is no dictionary
+ */
+ private boolean isNoDictionary;
+
+ /**
+ * @return the invertedIndexes
+ */
+ public int[] getInvertedIndexes() {
+ return invertedIndexes;
+ }
+
+ /**
+ * @param invertedIndexes the invertedIndexes to set
+ */
+ public void setInvertedIndexes(int[] invertedIndexes) {
+ this.invertedIndexes = invertedIndexes;
+ }
+
+ /**
+ * @return the invertedIndexesReverse
+ */
+ public int[] getInvertedIndexesReverse() {
+ return invertedIndexesReverse;
+ }
+
+ /**
+ * @param invertedIndexesReverse the invertedIndexesReverse to set
+ */
+ public void setInvertedIndexesReverse(int[] invertedIndexesReverse) {
+ this.invertedIndexesReverse = invertedIndexesReverse;
+ }
+
+ /**
+ * @return the eachRowSize
+ */
+ public int getColumnValueSize() {
+ return columnValueSize;
+ }
+
+ /**
+ * @param eachRowSize the eachRowSize to set
+ */
+ public void setEachRowSize(int eachRowSize) {
+ this.columnValueSize = eachRowSize;
+ }
+
+ /**
+ * @return the isNoDictionary
+ */
+ public boolean isNoDictionary() {
+ return isNoDictionary;
+ }
+
+ /**
+ * @param isNoDictionary the isNoDictionary to set
+ */
+ public void setNoDictionary(boolean isNoDictionary) {
+ this.isNoDictionary = isNoDictionary;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/DimensionColumnDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/DimensionColumnDataChunk.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/DimensionColumnDataChunk.java
new file mode 100644
index 0000000..ddc76c0
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/DimensionColumnDataChunk.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk;
+
+import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
+
+/**
+ * Interface for dimension column chunk.
+ */
+public interface DimensionColumnDataChunk<T> {
+
+ /**
+ * Below method will be used to fill the data based on offset and row id
+ *
+ * @param data data to filed
+ * @param offset offset from which data need to be filed
+ * @param rowId row id of the chunk
+ * @return how many bytes was copied
+ */
+ int fillChunkData(byte[] data, int offset, int columnIndex, KeyStructureInfo restructuringInfo);
+
+ /**
+ * It uses to convert column data to dictionary integer value
+ * @param rowId
+ * @param columnIndex
+ * @param row
+ * @param restructuringInfo @return
+ */
+ int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
+ KeyStructureInfo restructuringInfo);
+
+ /**
+ * Below method to get the data based in row id
+ *
+ * @param row id
+ * row id of the data
+ * @return chunk
+ */
+ byte[] getChunkData(int columnIndex);
+
+ /**
+ * Below method will be used get the chunk attributes
+ *
+ * @return chunk attributes
+ */
+ DimensionChunkAttributes getAttributes();
+
+ /**
+ * Below method will be used to return the complete data chunk
+ * This will be required during filter query
+ *
+ * @return complete chunk
+ */
+ T getCompleteDataChunk();
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/MeasureColumnDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/MeasureColumnDataChunk.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/MeasureColumnDataChunk.java
new file mode 100644
index 0000000..fbe6e95
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/MeasureColumnDataChunk.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk;
+
+import org.apache.carbondata.core.carbon.metadata.blocklet.datachunk.PresenceMeta;
+import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
+
+/**
+ * Holder for measure column chunk
+ * it will have data and its attributes which will
+ * be required for processing
+ */
+public class MeasureColumnDataChunk {
+
+ /**
+ * measure chunk
+ */
+ private CarbonReadDataHolder measureDataHolder;
+
+ /**
+ * below to hold null value holds this information
+ * about the null value index this will be helpful in case of
+ * to remove the null value while aggregation
+ */
+ private PresenceMeta nullValueIndexHolder;
+
+ /**
+ * @return the measureDataHolder
+ */
+ public CarbonReadDataHolder getMeasureDataHolder() {
+ return measureDataHolder;
+ }
+
+ /**
+ * @param measureDataHolder the measureDataHolder to set
+ */
+ public void setMeasureDataHolder(CarbonReadDataHolder measureDataHolder) {
+ this.measureDataHolder = measureDataHolder;
+ }
+
+ /**
+ * @return the nullValueIndexHolder
+ */
+ public PresenceMeta getNullValueIndexHolder() {
+ return nullValueIndexHolder;
+ }
+
+ /**
+ * @param nullValueIndexHolder the nullValueIndexHolder to set
+ */
+ public void setNullValueIndexHolder(PresenceMeta nullValueIndexHolder) {
+ this.nullValueIndexHolder = nullValueIndexHolder;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
new file mode 100644
index 0000000..77fb163
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/impl/ColumnGroupDimensionDataChunk.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.impl;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.DimensionChunkAttributes;
+import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
+
+/**
+ * This class is holder of the dimension column chunk data of the fixed length
+ * key size
+ */
+public class ColumnGroupDimensionDataChunk implements DimensionColumnDataChunk<byte[]> {
+
+ /**
+ * dimension chunk attributes
+ */
+ private DimensionChunkAttributes chunkAttributes;
+
+ /**
+ * data chunks
+ */
+ private byte[] dataChunk;
+
+ /**
+ * Constructor for this class
+ *
+ * @param dataChunk data chunk
+ * @param chunkAttributes chunk attributes
+ */
+ public ColumnGroupDimensionDataChunk(byte[] dataChunk, DimensionChunkAttributes chunkAttributes) {
+ this.chunkAttributes = chunkAttributes;
+ this.dataChunk = dataChunk;
+ }
+
+ /**
+ * Below method will be used to fill the data based on offset and row id
+ *
+ * @param data data to filed
+ * @param offset offset from which data need to be filed
+ * @param rowId row id of the chunk
+ * @param restructuringInfo define the structure of the key
+ * @return how many bytes was copied
+ */
+ @Override public int fillChunkData(byte[] data, int offset, int rowId,
+ KeyStructureInfo restructuringInfo) {
+ byte[] maskedKey =
+ getMaskedKey(dataChunk, rowId * chunkAttributes.getColumnValueSize(), restructuringInfo);
+ System.arraycopy(maskedKey, 0, data, offset, maskedKey.length);
+ return maskedKey.length;
+ }
+
+ /**
+ * Converts to column dictionary integer value
+ */
+ @Override public int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
+ KeyStructureInfo info) {
+ int start = rowId * chunkAttributes.getColumnValueSize();
+ long[] keyArray = info.getKeyGenerator().getKeyArray(dataChunk, start);
+ int[] ordinal = info.getMdkeyQueryDimensionOrdinal();
+ for (int i = 0; i < ordinal.length; i++) {
+ row[columnIndex++] = (int)keyArray[ordinal[i]];
+ }
+ return columnIndex;
+ }
+
+ /**
+ * Below method masks key
+ *
+ */
+ public byte[] getMaskedKey(byte[] data, int offset, KeyStructureInfo info) {
+ byte[] maskedKey = new byte[info.getMaskByteRanges().length];
+ int counter = 0;
+ int byteRange = 0;
+ for (int i = 0; i < info.getMaskByteRanges().length; i++) {
+ byteRange = info.getMaskByteRanges()[i];
+ maskedKey[counter++] = (byte) (data[byteRange + offset] & info.getMaxKey()[byteRange]);
+ }
+ return maskedKey;
+ }
+
+ /**
+ * Below method to get the data based in row id
+ *
+ * @param rowId row id of the data
+ * @return chunk
+ */
+ @Override public byte[] getChunkData(int rowId) {
+ byte[] data = new byte[chunkAttributes.getColumnValueSize()];
+ System.arraycopy(dataChunk, rowId * data.length, data, 0, data.length);
+ return data;
+ }
+
+ /**
+ * Below method will be used get the chunk attributes
+ *
+ * @return chunk attributes
+ */
+ @Override public DimensionChunkAttributes getAttributes() {
+ return chunkAttributes;
+ }
+
+ /**
+ * Below method will be used to return the complete data chunk
+ * This will be required during filter query
+ *
+ * @return complete chunk
+ */
+ @Override public byte[] getCompleteDataChunk() {
+ return dataChunk;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/impl/FixedLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
new file mode 100644
index 0000000..2867d76
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.impl;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.DimensionChunkAttributes;
+import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
+
+/**
+ * This class is holder of the dimension column chunk data of the fixed length
+ * key size
+ */
+public class FixedLengthDimensionDataChunk implements DimensionColumnDataChunk<byte[]> {
+
+ /**
+ * dimension chunk attributes
+ */
+ private DimensionChunkAttributes chunkAttributes;
+
+ /**
+ * data chunks
+ */
+ private byte[] dataChunk;
+
+ /**
+ * Constructor for this class
+ *
+ * @param dataChunk data chunk
+ * @param chunkAttributes chunk attributes
+ */
+ public FixedLengthDimensionDataChunk(byte[] dataChunk, DimensionChunkAttributes chunkAttributes) {
+ this.chunkAttributes = chunkAttributes;
+ this.dataChunk = dataChunk;
+ }
+
+ /**
+ * Below method will be used to fill the data based on offset and row id
+ *
+ * @param data data to filed
+ * @param offset offset from which data need to be filed
+ * @param index row id of the chunk
+ * @param keyStructureInfo define the structure of the key
+ * @return how many bytes was copied
+ */
+ @Override public int fillChunkData(byte[] data, int offset, int index,
+ KeyStructureInfo keyStructureInfo) {
+ if (chunkAttributes.getInvertedIndexes() != null) {
+ index = chunkAttributes.getInvertedIndexesReverse()[index];
+ }
+ System.arraycopy(dataChunk, index * chunkAttributes.getColumnValueSize(), data, offset,
+ chunkAttributes.getColumnValueSize());
+ return chunkAttributes.getColumnValueSize();
+ }
+
+ /**
+ * Converts to column dictionary integer value
+ */
+ @Override public int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
+ KeyStructureInfo restructuringInfo) {
+ if (chunkAttributes.getInvertedIndexes() != null) {
+ rowId = chunkAttributes.getInvertedIndexesReverse()[rowId];
+ }
+ int start = rowId * chunkAttributes.getColumnValueSize();
+ int dict = 0;
+ for (int i = start; i < start + chunkAttributes.getColumnValueSize(); i++) {
+ dict <<= 8;
+ dict ^= dataChunk[i] & 0xFF;
+ }
+ row[columnIndex] = dict;
+ return columnIndex + 1;
+ }
+
+ /**
+ * Below method to get the data based in row id
+ *
+ * @param index row id of the data
+ * @return chunk
+ */
+ @Override public byte[] getChunkData(int index) {
+ byte[] data = new byte[chunkAttributes.getColumnValueSize()];
+ if (chunkAttributes.getInvertedIndexes() != null) {
+ index = chunkAttributes.getInvertedIndexesReverse()[index];
+ }
+ System.arraycopy(dataChunk, index * chunkAttributes.getColumnValueSize(), data, 0,
+ chunkAttributes.getColumnValueSize());
+ return data;
+ }
+
+ /**
+ * Below method will be used get the chunk attributes
+ *
+ * @return chunk attributes
+ */
+ @Override public DimensionChunkAttributes getAttributes() {
+ return chunkAttributes;
+ }
+
+ /**
+ * Below method will be used to return the complete data chunk
+ * This will be required during filter query
+ *
+ * @return complete chunk
+ */
+ @Override public byte[] getCompleteDataChunk() {
+ return dataChunk;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/impl/VariableLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
new file mode 100644
index 0000000..6d2d400
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/impl/VariableLengthDimensionDataChunk.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.impl;
+
+import java.util.List;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.DimensionChunkAttributes;
+import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.scan.executor.infos.KeyStructureInfo;
+
+/**
+ * This class is holder of the dimension column chunk data of the variable
+ * length key size
+ */
+public class VariableLengthDimensionDataChunk implements DimensionColumnDataChunk<List<byte[]>> {
+
+ /**
+ * dimension chunk attributes
+ */
+ private DimensionChunkAttributes chunkAttributes;
+
+ /**
+ * data chunk
+ */
+ private List<byte[]> dataChunk;
+
+ /**
+ * Constructor for this class
+ *
+ * @param dataChunk data chunk
+ * @param chunkAttributes chunk attributes
+ */
+ public VariableLengthDimensionDataChunk(List<byte[]> dataChunk,
+ DimensionChunkAttributes chunkAttributes) {
+ this.chunkAttributes = chunkAttributes;
+ this.dataChunk = dataChunk;
+ }
+
+ /**
+ * Below method will be used to fill the data based on offset and row id
+ *
+ * @param data data to filed
+ * @param offset offset from which data need to be filed
+ * @param index row id of the chunk
+ * @param restructuringInfo define the structure of the key
+ * @return how many bytes was copied
+ */
+ @Override public int fillChunkData(byte[] data, int offset, int index,
+ KeyStructureInfo restructuringInfo) {
+ // no required in this case because this column chunk is not the part if
+ // mdkey
+ return 0;
+ }
+
+ /**
+ * Converts to column dictionary integer value
+ * @param rowId
+ * @param columnIndex
+ * @param row
+ * @param restructuringInfo @return
+ */
+ @Override public int fillConvertedChunkData(int rowId, int columnIndex, int[] row,
+ KeyStructureInfo restructuringInfo) {
+ return columnIndex + 1;
+ }
+
+ /**
+ * Below method to get the data based in row id
+ *
+ * @param index row id of the data
+ * @return chunk
+ */
+ @Override public byte[] getChunkData(int index) {
+ if (null != chunkAttributes.getInvertedIndexes()) {
+ index = chunkAttributes.getInvertedIndexesReverse()[index];
+ }
+ return dataChunk.get(index);
+ }
+
+ /**
+ * Below method will be used get the chunk attributes
+ *
+ * @return chunk attributes
+ */
+ @Override public DimensionChunkAttributes getAttributes() {
+ return chunkAttributes;
+ }
+
+ /**
+ * Below method will be used to return the complete data chunk
+ * This will be required during filter query
+ *
+ * @return complete chunk
+ */
+ @Override public List<byte[]> getCompleteDataChunk() {
+ return dataChunk;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/DimensionColumnChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/DimensionColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/DimensionColumnChunkReader.java
new file mode 100644
index 0000000..b958245
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/DimensionColumnChunkReader.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.reader;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.datastorage.store.FileHolder;
+
+/**
+ * Interface for reading the data chunk
+ * Its concrete implementation can be used to read the chunk.
+ * compressed or uncompressed chunk
+ */
+public interface DimensionColumnChunkReader {
+
+ /**
+ * Below method will be used to read the chunk based on block indexes
+ *
+ * @param fileReader file reader to read the blocks from file
+ * @param blockIndexes blocks to be read
+ * @return dimension column chunks
+ */
+ DimensionColumnDataChunk[] readDimensionChunks(FileHolder fileReader, int... blockIndexes);
+
+ /**
+ * Below method will be used to read the chunk based on block index
+ *
+ * @param fileReader file reader to read the blocks from file
+ * @param blockIndex block to be read
+ * @return dimension column chunk
+ */
+ DimensionColumnDataChunk readDimensionChunk(FileHolder fileReader, int blockIndex);
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/MeasureColumnChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/MeasureColumnChunkReader.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/MeasureColumnChunkReader.java
new file mode 100644
index 0000000..8a7c8ef
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/MeasureColumnChunkReader.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.reader;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.MeasureColumnDataChunk;
+import org.apache.carbondata.core.datastorage.store.FileHolder;
+
+/**
+ * Reader interface for reading the measure blocks from file
+ */
+public interface MeasureColumnChunkReader {
+
+ /**
+ * Method to read the blocks data based on block indexes
+ *
+ * @param fileReader file reader to read the blocks
+ * @param blockIndexes blocks to be read
+ * @return measure data chunks
+ */
+ MeasureColumnDataChunk[] readMeasureChunks(FileHolder fileReader, int... blockIndexes);
+
+ /**
+ * Method to read the blocks data based on block index
+ *
+ * @param fileReader file reader to read the blocks
+ * @param blockIndex block to be read
+ * @return measure data chunk
+ */
+ MeasureColumnDataChunk readMeasureChunk(FileHolder fileReader, int blockIndex);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/AbstractChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/AbstractChunkReader.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/AbstractChunkReader.java
new file mode 100644
index 0000000..59dcd38
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/AbstractChunkReader.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.reader.dimension;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.reader.DimensionColumnChunkReader;
+import org.apache.carbondata.core.carbon.metadata.blocklet.datachunk.DataChunk;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastorage.store.compression.Compressor;
+import org.apache.carbondata.core.datastorage.store.compression.SnappyCompression;
+import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+/**
+ * Class which will have all the common properties and behavior among all type
+ * of reader
+ */
+public abstract class AbstractChunkReader implements DimensionColumnChunkReader {
+
+ /**
+ * compressor will be used to uncompress the data
+ */
+ protected static final Compressor<byte[]> COMPRESSOR =
+ SnappyCompression.SnappyByteCompression.INSTANCE;
+
+ /**
+ * data chunk list which holds the information
+ * about the data block metadata
+ */
+ protected List<DataChunk> dimensionColumnChunk;
+
+ /**
+ * size of the each column value
+ * for no dictionary column it will be -1
+ */
+ protected int[] eachColumnValueSize;
+
+ /**
+ * full qualified path of the data file from
+ * which data will be read
+ */
+ protected String filePath;
+
+ /**
+ * this will be used to uncompress the
+ * row id and rle chunk
+ */
+ protected NumberCompressor numberComressor;
+
+ /**
+ * number of element in each chunk
+ */
+ private int numberOfElement;
+
+ /**
+ * Constructor to get minimum parameter to create
+ * instance of this class
+ *
+ * @param dimensionColumnChunk dimension chunk metadata
+ * @param eachColumnValueSize size of the each column value
+ * @param filePath file from which data will be read
+ */
+ public AbstractChunkReader(List<DataChunk> dimensionColumnChunk, int[] eachColumnValueSize,
+ String filePath) {
+ this.dimensionColumnChunk = dimensionColumnChunk;
+ this.eachColumnValueSize = eachColumnValueSize;
+ this.filePath = filePath;
+ int numberOfElement = 0;
+ try {
+ numberOfElement = Integer.parseInt(CarbonProperties.getInstance()
+ .getProperty(CarbonCommonConstants.BLOCKLET_SIZE,
+ CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL));
+ } catch (NumberFormatException exception) {
+ numberOfElement = Integer.parseInt(CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL);
+ }
+ this.numberComressor = new NumberCompressor(numberOfElement);
+ }
+
+ /**
+ * Below method will be used to create the inverted index reverse
+ * this will be used to point to actual data in the chunk
+ *
+ * @param invertedIndex inverted index
+ * @return reverse inverted index
+ */
+ protected int[] getInvertedReverseIndex(int[] invertedIndex) {
+ int[] columnIndexTemp = new int[invertedIndex.length];
+
+ for (int i = 0; i < invertedIndex.length; i++) {
+ columnIndexTemp[invertedIndex[i]] = i;
+ }
+ return columnIndexTemp;
+ }
+
+ /**
+ * In case of no dictionary column size of the each column value
+ * will not be same, so in case of filter query we can not take
+ * advantage of binary search as length with each value will be also
+ * store with the data, so converting this data to two dimension
+ * array format filter query processing will be faster
+ *
+ * @param dataChunkWithLength no dictionary column chunk
+ * <Lenght><Data><Lenght><data>
+ * Length will store in 2 bytes
+ * @return list of data chuck, one value in list will represent one column value
+ */
+ protected List<byte[]> getNoDictionaryDataChunk(byte[] dataChunkWithLength) {
+ List<byte[]> dataChunk = new ArrayList<byte[]>(numberOfElement);
+ // wrapping the chunk to byte buffer
+ ByteBuffer buffer = ByteBuffer.wrap(dataChunkWithLength);
+ buffer.rewind();
+ byte[] data = null;
+ // iterating till all the elements are read
+ while (buffer.hasRemaining()) {
+ // as all the data is stored with length(2 bytes)
+ // first reading the size and then based on size
+ // we need to read the actual value
+ data = new byte[buffer.getShort()];
+ buffer.get(data);
+ dataChunk.add(data);
+ }
+ return dataChunk;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/CompressedDimensionChunkFileBasedReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/CompressedDimensionChunkFileBasedReader.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/CompressedDimensionChunkFileBasedReader.java
new file mode 100644
index 0000000..209217b
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/dimension/CompressedDimensionChunkFileBasedReader.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.reader.dimension;
+
+import java.util.List;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.DimensionChunkAttributes;
+import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.carbon.datastore.chunk.impl.ColumnGroupDimensionDataChunk;
+import org.apache.carbondata.core.carbon.datastore.chunk.impl.FixedLengthDimensionDataChunk;
+import org.apache.carbondata.core.carbon.datastore.chunk.impl.VariableLengthDimensionDataChunk;
+import org.apache.carbondata.core.carbon.metadata.blocklet.datachunk.DataChunk;
+import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
+import org.apache.carbondata.core.datastorage.store.FileHolder;
+import org.apache.carbondata.core.datastorage.store.columnar.UnBlockIndexer;
+import org.apache.carbondata.core.util.CarbonUtil;
+
+/**
+ * Compressed dimension chunk reader class
+ */
+public class CompressedDimensionChunkFileBasedReader extends AbstractChunkReader {
+
+ /**
+ * Constructor to get minimum parameter to create instance of this class
+ *
+ * @param dimensionColumnChunk dimension chunk metadata
+ * @param eachColumnValueSize size of the each column value
+ * @param filePath file from which data will be read
+ */
+ public CompressedDimensionChunkFileBasedReader(List<DataChunk> dimensionColumnChunk,
+ int[] eachColumnValueSize, String filePath) {
+ super(dimensionColumnChunk, eachColumnValueSize, filePath);
+ }
+
+ /**
+ * Below method will be used to read the chunk based on block indexes
+ *
+ * @param fileReader file reader to read the blocks from file
+ * @param blockIndexes blocks to be read
+ * @return dimension column chunks
+ */
+ @Override public DimensionColumnDataChunk[] readDimensionChunks(FileHolder fileReader,
+ int... blockIndexes) {
+ // read the column chunk based on block index and add
+ DimensionColumnDataChunk[] dataChunks =
+ new DimensionColumnDataChunk[dimensionColumnChunk.size()];
+ for (int i = 0; i < blockIndexes.length; i++) {
+ dataChunks[blockIndexes[i]] = readDimensionChunk(fileReader, blockIndexes[i]);
+ }
+ return dataChunks;
+ }
+
+ /**
+ * Below method will be used to read the chunk based on block index
+ *
+ * @param fileReader file reader to read the blocks from file
+ * @param blockIndex block to be read
+ * @return dimension column chunk
+ */
+ @Override public DimensionColumnDataChunk readDimensionChunk(FileHolder fileReader,
+ int blockIndex) {
+ byte[] dataPage = null;
+ int[] invertedIndexes = null;
+ int[] invertedIndexesReverse = null;
+ int[] rlePage = null;
+
+ // first read the data and uncompressed it
+ dataPage = COMPRESSOR.unCompress(fileReader
+ .readByteArray(filePath, dimensionColumnChunk.get(blockIndex).getDataPageOffset(),
+ dimensionColumnChunk.get(blockIndex).getDataPageLength()));
+ // if row id block is present then read the row id chunk and uncompress it
+ if (CarbonUtil.hasEncoding(dimensionColumnChunk.get(blockIndex).getEncodingList(),
+ Encoding.INVERTED_INDEX)) {
+ invertedIndexes = CarbonUtil
+ .getUnCompressColumnIndex(dimensionColumnChunk.get(blockIndex).getRowIdPageLength(),
+ fileReader.readByteArray(filePath,
+ dimensionColumnChunk.get(blockIndex).getRowIdPageOffset(),
+ dimensionColumnChunk.get(blockIndex).getRowIdPageLength()), numberComressor);
+ // get the reverse index
+ invertedIndexesReverse = getInvertedReverseIndex(invertedIndexes);
+ }
+ // if rle is applied then read the rle block chunk and then uncompress
+ //then actual data based on rle block
+ if (CarbonUtil
+ .hasEncoding(dimensionColumnChunk.get(blockIndex).getEncodingList(), Encoding.RLE)) {
+ // read and uncompress the rle block
+ rlePage = numberComressor.unCompress(fileReader
+ .readByteArray(filePath, dimensionColumnChunk.get(blockIndex).getRlePageOffset(),
+ dimensionColumnChunk.get(blockIndex).getRlePageLength()));
+ // uncompress the data with rle indexes
+ dataPage = UnBlockIndexer.uncompressData(dataPage, rlePage, eachColumnValueSize[blockIndex]);
+ rlePage = null;
+ }
+ // fill chunk attributes
+ DimensionChunkAttributes chunkAttributes = new DimensionChunkAttributes();
+ chunkAttributes.setEachRowSize(eachColumnValueSize[blockIndex]);
+ chunkAttributes.setInvertedIndexes(invertedIndexes);
+ chunkAttributes.setInvertedIndexesReverse(invertedIndexesReverse);
+ DimensionColumnDataChunk columnDataChunk = null;
+
+ if (dimensionColumnChunk.get(blockIndex).isRowMajor()) {
+ // to store fixed length column chunk values
+ columnDataChunk = new ColumnGroupDimensionDataChunk(dataPage, chunkAttributes);
+ }
+ // if no dictionary column then first create a no dictionary column chunk
+ // and set to data chunk instance
+ else if (!CarbonUtil
+ .hasEncoding(dimensionColumnChunk.get(blockIndex).getEncodingList(), Encoding.DICTIONARY)) {
+ columnDataChunk =
+ new VariableLengthDimensionDataChunk(getNoDictionaryDataChunk(dataPage), chunkAttributes);
+ chunkAttributes.setNoDictionary(true);
+ } else {
+ // to store fixed length column chunk values
+ columnDataChunk = new FixedLengthDimensionDataChunk(dataPage, chunkAttributes);
+ }
+ return columnDataChunk;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/AbstractMeasureChunkReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/AbstractMeasureChunkReader.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/AbstractMeasureChunkReader.java
new file mode 100644
index 0000000..dc8771f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/AbstractMeasureChunkReader.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.reader.measure;
+
+import java.util.List;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.reader.MeasureColumnChunkReader;
+import org.apache.carbondata.core.carbon.metadata.blocklet.datachunk.DataChunk;
+import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
+import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
+import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder.UnCompressValue;
+
+/**
+ * Measure block reader abstract class
+ */
+public abstract class AbstractMeasureChunkReader implements MeasureColumnChunkReader {
+
+ /**
+ * metadata which was to used to compress and uncompress the measure value
+ */
+ protected ValueCompressionModel compressionModel;
+
+ /**
+ * file path from which blocks will be read
+ */
+ protected String filePath;
+
+ /**
+ * measure chunk have the information about the metadata present in the file
+ */
+ protected List<DataChunk> measureColumnChunk;
+
+ /**
+ * type of valu comprssion model selected for each measure column
+ */
+ protected UnCompressValue[] values;
+
+ /**
+ * Constructor to get minimum parameter to create instance of this class
+ *
+ * @param measureColumnChunk measure chunk metadata
+ * @param compression model metadata which was to used to compress and uncompress
+ * the measure value
+ * @param filePath file from which data will be read
+ * @param isInMemory in case of in memory it will read and holds the data and when
+ * query request will come it will uncompress and the data
+ */
+ public AbstractMeasureChunkReader(List<DataChunk> measureColumnChunk,
+ ValueCompressionModel compressionModel, String filePath, boolean isInMemory) {
+ this.measureColumnChunk = measureColumnChunk;
+ this.compressionModel = compressionModel;
+ this.filePath = filePath;
+ values =
+ new ValueCompressonHolder.UnCompressValue[compressionModel.getUnCompressValues().length];
+ for (int i = 0; i < values.length; i++) {
+ values[i] = compressionModel.getUnCompressValues()[i].getNew().getCompressorObject();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReader.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReader.java
new file mode 100644
index 0000000..31c470d
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/chunk/reader/measure/CompressedMeasureChunkFileBasedReader.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.chunk.reader.measure;
+
+import java.util.List;
+
+import org.apache.carbondata.core.carbon.datastore.chunk.MeasureColumnDataChunk;
+import org.apache.carbondata.core.carbon.metadata.blocklet.datachunk.DataChunk;
+import org.apache.carbondata.core.datastorage.store.FileHolder;
+import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
+import org.apache.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
+import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
+
+/**
+ * Compressed measure chunk reader
+ */
+public class CompressedMeasureChunkFileBasedReader extends AbstractMeasureChunkReader {
+
+ /**
+ * Constructor to get minimum parameter to create instance of this class
+ *
+ * @param measureColumnChunk measure chunk metadata
+ * @param compression model metadata which was to used to compress and uncompress
+ * the measure value
+ * @param filePath file from which data will be read
+ */
+ public CompressedMeasureChunkFileBasedReader(List<DataChunk> measureColumnChunk,
+ ValueCompressionModel compressionModel, String filePath) {
+ super(measureColumnChunk, compressionModel, filePath, false);
+ }
+
+ /**
+ * Method to read the blocks data based on block indexes
+ *
+ * @param fileReader file reader to read the blocks
+ * @param blockIndexes blocks to be read
+ * @return measure data chunks
+ */
+ @Override public MeasureColumnDataChunk[] readMeasureChunks(FileHolder fileReader,
+ int... blockIndexes) {
+ MeasureColumnDataChunk[] datChunk = new MeasureColumnDataChunk[values.length];
+ for (int i = 0; i < blockIndexes.length; i++) {
+ datChunk[blockIndexes[i]] = readMeasureChunk(fileReader, blockIndexes[i]);
+ }
+ return datChunk;
+ }
+
+ /**
+ * Method to read the blocks data based on block index
+ *
+ * @param fileReader file reader to read the blocks
+ * @param blockIndex block to be read
+ * @return measure data chunk
+ */
+ @Override public MeasureColumnDataChunk readMeasureChunk(FileHolder fileReader, int blockIndex) {
+ MeasureColumnDataChunk datChunk = new MeasureColumnDataChunk();
+ // create a new uncompressor
+ ValueCompressonHolder.UnCompressValue copy = values[blockIndex].getNew();
+ // read data from file and set to uncompressor
+ copy.setValue(fileReader
+ .readByteArray(filePath, measureColumnChunk.get(blockIndex).getDataPageOffset(),
+ measureColumnChunk.get(blockIndex).getDataPageLength()));
+ // get the data holder after uncompressing
+ CarbonReadDataHolder measureDataHolder =
+ copy.uncompress(compressionModel.getChangedDataType()[blockIndex])
+ .getValues(compressionModel.getDecimal()[blockIndex],
+ compressionModel.getMaxValue()[blockIndex]);
+ // set the data chunk
+ datChunk.setMeasureDataHolder(measureDataHolder);
+ // set the enun value indexes
+ datChunk
+ .setNullValueIndexHolder(measureColumnChunk.get(blockIndex).getNullValueIndexForColumn());
+ return datChunk;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/exception/IndexBuilderException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/exception/IndexBuilderException.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/exception/IndexBuilderException.java
new file mode 100644
index 0000000..a9c7343
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/exception/IndexBuilderException.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.carbon.datastore.exception;
+
+import java.util.Locale;
+
+/**
+ * Exception class for block builder
+ *
+ * @author Administrator
+ */
+public class IndexBuilderException extends Exception {
+ /**
+ * default serial version ID.
+ */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * The Error message.
+ */
+ private String msg = "";
+
+ /**
+ * Constructor
+ *
+ * @param errorCode The error code for this exception.
+ * @param msg The error message for this exception.
+ */
+ public IndexBuilderException(String msg) {
+ super(msg);
+ this.msg = msg;
+ }
+
+ /**
+ * Constructor
+ *
+ * @param msg exception message
+ * @param throwable detail exception
+ */
+ public IndexBuilderException(String msg, Throwable throwable) {
+ super(msg, throwable);
+ this.msg = msg;
+ }
+
+ /**
+ * Constructor
+ *
+ * @param throwable exception
+ */
+ public IndexBuilderException(Throwable throwable) {
+ super(throwable);
+ }
+
+ /**
+ * This method is used to get the localized message.
+ *
+ * @param locale - A Locale object represents a specific geographical,
+ * political, or cultural region.
+ * @return - Localized error message.
+ */
+ public String getLocalizedMessage(Locale locale) {
+ return "";
+ }
+
+ /**
+ * getLocalizedMessage
+ */
+ @Override public String getLocalizedMessage() {
+ return super.getLocalizedMessage();
+ }
+
+ /**
+ * getMessage
+ */
+ public String getMessage() {
+ return this.msg;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeBuilder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeBuilder.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeBuilder.java
new file mode 100644
index 0000000..d414d3e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeBuilder.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.impl.btree;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.core.carbon.datastore.BtreeBuilder;
+import org.apache.carbondata.core.carbon.datastore.IndexKey;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.CarbonProperties;
+
+/**
+ * Abstract Btree based builder
+ */
+public abstract class AbstractBTreeBuilder implements BtreeBuilder {
+
+ /**
+ * default Number of keys per page
+ */
+ private static final int DEFAULT_NUMBER_OF_ENTRIES_NONLEAF = 32;
+
+ /**
+ * Maximum number of entries in intermediate nodes
+ */
+ protected int maxNumberOfEntriesInNonLeafNodes;
+
+ /**
+ * Number of leaf nodes
+ */
+ protected int nLeaf;
+
+ /**
+ * root node of a btree
+ */
+ protected BTreeNode root;
+
+ public AbstractBTreeBuilder() {
+ maxNumberOfEntriesInNonLeafNodes = Integer.parseInt(CarbonProperties.getInstance()
+ .getProperty("com.huawei.datastore.internalnodesize",
+ DEFAULT_NUMBER_OF_ENTRIES_NONLEAF + ""));
+ }
+
+ /**
+ * Below method is to build the intermediate node of the btree
+ *
+ * @param curNode current node
+ * @param childNodeGroups children group which will have all the children for
+ * particular intermediate node
+ * @param currentGroup current group
+ * @param interNSKeyList list if keys
+ * @param numberOfInternalNode number of internal node
+ */
+ protected void addIntermediateNode(BTreeNode curNode, List<BTreeNode[]> childNodeGroups,
+ BTreeNode[] currentGroup, List<List<IndexKey>> interNSKeyList, int numberOfInternalNode) {
+
+ int groupCounter;
+ // Build internal nodes level by level. Each upper node can have
+ // upperMaxEntry keys and upperMaxEntry+1 children
+ int remainder;
+ int nHigh = numberOfInternalNode;
+ boolean bRootBuilt = false;
+ remainder = nLeaf % (maxNumberOfEntriesInNonLeafNodes);
+ List<IndexKey> interNSKeys = null;
+ while (nHigh > 1 || !bRootBuilt) {
+ List<BTreeNode[]> internalNodeGroups =
+ new ArrayList<BTreeNode[]>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ List<List<IndexKey>> interNSKeyTmpList =
+ new ArrayList<List<IndexKey>>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ numberOfInternalNode = 0;
+ for (int i = 0; i < nHigh; i++) {
+ // Create a new internal node
+ curNode = new BTreeNonLeafNode();
+ // Allocate a new node group if current node group is full
+ groupCounter = i % (maxNumberOfEntriesInNonLeafNodes);
+ if (groupCounter == 0) {
+ // Create new node group
+ currentGroup = new BTreeNonLeafNode[maxNumberOfEntriesInNonLeafNodes];
+ internalNodeGroups.add(currentGroup);
+ numberOfInternalNode++;
+ interNSKeys = new ArrayList<IndexKey>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ interNSKeyTmpList.add(interNSKeys);
+ }
+
+ // Add the new internal node to current group
+ if (null != currentGroup) {
+ currentGroup[groupCounter] = curNode;
+ }
+ int nNodes;
+
+ if (i == nHigh - 1 && remainder != 0) {
+ nNodes = remainder;
+ } else {
+ nNodes = maxNumberOfEntriesInNonLeafNodes;
+ }
+ // Point the internal node to its children node group
+ curNode.setChildren(childNodeGroups.get(i));
+ // Fill the internal node with keys based on its child nodes
+ for (int j = 0; j < nNodes; j++) {
+ curNode.setKey(interNSKeyList.get(i).get(j));
+ if (j == 0 && null != interNSKeys) {
+ interNSKeys.add(interNSKeyList.get(i).get(j));
+
+ }
+ }
+ }
+ // If nHigh is 1, we have the root node
+ if (nHigh == 1) {
+ bRootBuilt = true;
+ }
+
+ remainder = nHigh % (maxNumberOfEntriesInNonLeafNodes);
+ nHigh = numberOfInternalNode;
+ childNodeGroups = internalNodeGroups;
+ interNSKeyList = interNSKeyTmpList;
+ }
+ root = curNode;
+ }
+
+ /**
+ * Below method is to convert the start key
+ * into fixed and variable length key.
+ * data format<lenght><fixed length key><length><variable length key>
+ *
+ * @param startKey
+ * @return Index key
+ */
+ protected IndexKey convertStartKeyToNodeEntry(byte[] startKey) {
+ ByteBuffer buffer = ByteBuffer.wrap(startKey);
+ buffer.rewind();
+ int dictonaryKeySize = buffer.getInt();
+ int nonDictonaryKeySize = buffer.getInt();
+ byte[] dictionaryKey = new byte[dictonaryKeySize];
+ buffer.get(dictionaryKey);
+ byte[] nonDictionaryKey = new byte[nonDictonaryKeySize];
+ buffer.get(nonDictionaryKey);
+ IndexKey entry = new IndexKey(dictionaryKey, nonDictionaryKey);
+ return entry;
+ }
+
+ /**
+ * Below method will be used to get the first data block
+ * in Btree case it will be root node
+ */
+ @Override public BTreeNode get() {
+ return root;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeLeafNode.java
new file mode 100644
index 0000000..de476ad
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeLeafNode.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.impl.btree;
+
+import org.apache.carbondata.core.carbon.datastore.DataRefNode;
+import org.apache.carbondata.core.carbon.datastore.IndexKey;
+import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk;
+import org.apache.carbondata.core.carbon.datastore.chunk.MeasureColumnDataChunk;
+import org.apache.carbondata.core.datastorage.store.FileHolder;
+
+/**
+ * Non leaf node abstract class
+ */
+public abstract class AbstractBTreeLeafNode implements BTreeNode {
+
+ /**
+ * number of keys in a btree
+ */
+ protected int numberOfKeys;
+
+ /**
+ * node number
+ */
+ protected long nodeNumber;
+
+ /**
+ * Next node of the leaf
+ */
+ protected BTreeNode nextNode;
+
+ /**
+ * max key of the column this will be used to check whether this leaf will
+ * be used for scanning or not
+ */
+ protected byte[][] maxKeyOfColumns;
+
+ /**
+ * min key of the column this will be used to check whether this leaf will
+ * be used for scanning or not
+ */
+ protected byte[][] minKeyOfColumns;
+
+ /**
+ * Method to get the next block this can be used while scanning when
+ * iterator of this class can be used iterate over blocks
+ *
+ * @return next block
+ */
+ @Override public int nodeSize() {
+ return this.numberOfKeys;
+ }
+
+ /**
+ * below method will used to set the next node
+ *
+ * @param nextNode
+ */
+ @Override public void setNextNode(BTreeNode nextNode) {
+ this.nextNode = nextNode;
+ }
+
+ /**
+ * Below method is to get the children based on index
+ *
+ * @param index children index
+ * @return btree node
+ */
+ @Override public BTreeNode getChild(int index) {
+ throw new UnsupportedOperationException("Operation not supported in case of leaf node");
+ }
+
+ /**
+ * below method to set the node entry
+ *
+ * @param key node entry
+ */
+ @Override public void setKey(IndexKey key) {
+ throw new UnsupportedOperationException("Operation not supported in case of leaf node");
+ }
+
+ /**
+ * Method can be used to get the block index .This can be used when multiple
+ * thread can be used scan group of blocks in that can we can assign the
+ * some of the blocks to one thread and some to other
+ *
+ * @return block number
+ */
+ @Override public long nodeNumber() {
+ return nodeNumber;
+ }
+
+ /**
+ * This method will be used to get the max value of all the columns this can
+ * be used in case of filter query
+ *
+ * @param max value of all the columns
+ */
+ @Override public byte[][] getColumnsMaxValue() {
+ return maxKeyOfColumns;
+ }
+
+ /**
+ * This method will be used to get the max value of all the columns this can
+ * be used in case of filter query
+ *
+ * @param max value of all the columns
+ */
+ @Override public byte[][] getColumnsMinValue() {
+ return minKeyOfColumns;
+ }
+
+ /**
+ * to check whether node in a btree is a leaf node or not
+ *
+ * @return leaf node or not
+ */
+ @Override public boolean isLeafNode() {
+ return true;
+ }
+
+ /**
+ * Method to get the next block this can be used while scanning when
+ * iterator of this class can be used iterate over blocks
+ *
+ * @return next block
+ */
+ @Override public DataRefNode getNextDataRefNode() {
+ return nextNode;
+ }
+
+ /**
+ * below method will return the one node indexes
+ *
+ * @return node entry array
+ */
+ @Override public IndexKey[] getNodeKeys() {
+ // as this is a leaf node so this method implementation is not required
+ throw new UnsupportedOperationException("Operation not supported in case of leaf node");
+ }
+
+ /**
+ * below method will be used to set the children of intermediate node
+ *
+ * @param children array
+ */
+ @Override public void setChildren(BTreeNode[] children) {
+ // no required in case of leaf node as leaf node will not have any children
+ throw new UnsupportedOperationException("Operation not supported in case of leaf node");
+ }
+
+ /**
+ * Below method will be used to get the dimension chunks
+ *
+ * @param fileReader file reader to read the chunks from file
+ * @param blockIndexes indexes of the blocks need to be read
+ * @return dimension data chunks
+ */
+ @Override public DimensionColumnDataChunk[] getDimensionChunks(FileHolder fileReader,
+ int[] blockIndexes) {
+ // No required here as leaf which will will be use this class will implement its own get
+ // dimension chunks
+ return null;
+ }
+
+ /**
+ * Below method will be used to get the dimension chunk
+ *
+ * @param fileReader file reader to read the chunk from file
+ * @param blockIndex block index to be read
+ * @return dimension data chunk
+ */
+ @Override public DimensionColumnDataChunk getDimensionChunk(FileHolder fileReader,
+ int blockIndex) {
+ // No required here as leaf which will will be use this class will implement
+ // its own get dimension chunks
+ return null;
+ }
+
+ /**
+ * Below method will be used to get the measure chunk
+ *
+ * @param fileReader file reader to read the chunk from file
+ * @param blockIndexes block indexes to be read from file
+ * @return measure column data chunk
+ */
+ @Override public MeasureColumnDataChunk[] getMeasureChunks(FileHolder fileReader,
+ int[] blockIndexes) {
+ // No required here as leaf which will will be use this class will implement its own get
+ // measure chunks
+ return null;
+ }
+
+ /**
+ * Below method will be used to read the measure chunk
+ *
+ * @param fileReader file read to read the file chunk
+ * @param blockIndex block index to be read from file
+ * @return measure data chunk
+ */
+ @Override public MeasureColumnDataChunk getMeasureChunk(FileHolder fileReader, int blockIndex) {
+ // No required here as leaf which will will be use this class will implement its own get
+ // measure chunks
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeDataRefNodeFinder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeDataRefNodeFinder.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeDataRefNodeFinder.java
new file mode 100644
index 0000000..e443182
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeDataRefNodeFinder.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.impl.btree;
+
+import java.nio.ByteBuffer;
+
+import org.apache.carbondata.core.carbon.datastore.DataRefNode;
+import org.apache.carbondata.core.carbon.datastore.DataRefNodeFinder;
+import org.apache.carbondata.core.carbon.datastore.IndexKey;
+import org.apache.carbondata.core.util.ByteUtil;
+
+/**
+ * Below class will be used to find a block in a btree
+ */
+public class BTreeDataRefNodeFinder implements DataRefNodeFinder {
+
+ /**
+ * no dictionary column value is of variable length so in each column value
+ * it will -1
+ */
+ private static final int NO_DCITIONARY_COLUMN_VALUE = -1;
+
+ /**
+ * sized of the short value in bytes
+ */
+ private static final short SHORT_SIZE_IN_BYTES = 2;
+ /**
+ * this will holds the information about the size of each value of a column,
+ * this will be used during Comparison of the btree node value and the
+ * search value if value is more than zero then its a fixed length column
+ * else its variable length column. So as data of both type of column store
+ * separately so this value size array will be used for both purpose
+ * comparison and jumping(which type value we need to compare)
+ */
+ private int[] eachColumnValueSize;
+
+ /**
+ * this will be used during search for no dictionary column
+ */
+ private int numberOfNoDictionaryColumns;
+
+ public BTreeDataRefNodeFinder(int[] eachColumnValueSize) {
+ this.eachColumnValueSize = eachColumnValueSize;
+
+ for (int i = 0; i < eachColumnValueSize.length; i++) {
+ if (eachColumnValueSize[i] == -1) {
+ numberOfNoDictionaryColumns++;
+ }
+ }
+ }
+
+ /**
+ * Below method will be used to get the first tentative data block based on
+ * search key
+ *
+ * @param dataBlocks complete data blocks present
+ * @param serachKey key to be search
+ * @return data block
+ */
+ @Override public DataRefNode findFirstDataBlock(DataRefNode dataRefBlock, IndexKey searchKey) {
+ // as its for btree type cast it to btree interface
+ BTreeNode rootNode = (BTreeNode) dataRefBlock;
+ while (!rootNode.isLeafNode()) {
+ rootNode = findFirstLeafNode(searchKey, rootNode);
+ }
+ return rootNode;
+ }
+
+ /**
+ * Below method will be used to get the last data tentative block based on
+ * search key
+ *
+ * @param dataBlocks complete data blocks present
+ * @param serachKey key to be search
+ * @return data block
+ */
+ @Override public DataRefNode findLastDataBlock(DataRefNode dataRefBlock, IndexKey searchKey) {
+ // as its for btree type cast it to btree interface
+ BTreeNode rootNode = (BTreeNode) dataRefBlock;
+ while (!rootNode.isLeafNode()) {
+ rootNode = findLastLeafNode(searchKey, rootNode);
+ }
+ return rootNode;
+ }
+
+ /**
+ * Binary search used to get the first tentative block of the btree based on
+ * search key
+ *
+ * @param key search key
+ * @param node root node of btree
+ * @return first tentative block
+ */
+ private BTreeNode findFirstLeafNode(IndexKey key, BTreeNode node) {
+ int childNodeIndex;
+ int low = 0;
+ int high = node.nodeSize() - 1;
+ int mid = 0;
+ int compareRes = -1;
+ IndexKey[] nodeKeys = node.getNodeKeys();
+ //
+ while (low <= high) {
+ mid = (low + high) >>> 1;
+ // compare the entries
+ compareRes = compareIndexes(key, nodeKeys[mid]);
+ if (compareRes < 0) {
+ high = mid - 1;
+ } else if (compareRes > 0) {
+ low = mid + 1;
+ } else {
+ // if key is matched then get the first entry
+ int currentPos = mid;
+ while (currentPos - 1 >= 0 && compareIndexes(key, nodeKeys[currentPos - 1]) == 0) {
+ currentPos--;
+ }
+ mid = currentPos;
+ break;
+ }
+ }
+ // if compare result is less than zero then we
+ // and mid is more than 0 then we need to previous block as duplicates
+ // record can be present
+ if (compareRes < 0) {
+ if (mid > 0) {
+ mid--;
+ }
+ childNodeIndex = mid;
+ } else {
+ childNodeIndex = mid;
+ }
+ // get the leaf child
+ node = node.getChild(childNodeIndex);
+ return node;
+ }
+
+ /**
+ * Binary search used to get the last tentative block of the btree based on
+ * search key
+ *
+ * @param key search key
+ * @param node root node of btree
+ * @return first tentative block
+ */
+ private BTreeNode findLastLeafNode(IndexKey key, BTreeNode node) {
+ int childNodeIndex;
+ int low = 0;
+ int high = node.nodeSize() - 1;
+ int mid = 0;
+ int compareRes = -1;
+ IndexKey[] nodeKeys = node.getNodeKeys();
+ //
+ while (low <= high) {
+ mid = (low + high) >>> 1;
+ // compare the entries
+ compareRes = compareIndexes(key, nodeKeys[mid]);
+ if (compareRes < 0) {
+ high = mid - 1;
+ } else if (compareRes > 0) {
+ low = mid + 1;
+ } else {
+ int currentPos = mid;
+ // if key is matched then get the first entry
+ while (currentPos + 1 < node.nodeSize()
+ && compareIndexes(key, nodeKeys[currentPos + 1]) == 0) {
+ currentPos++;
+ }
+ mid = currentPos;
+ break;
+ }
+ }
+ // if compare result is less than zero then we
+ // and mid is more than 0 then we need to previous block as duplicates
+ // record can be present
+ if (compareRes < 0) {
+ if (mid > 0) {
+ mid--;
+ }
+ childNodeIndex = mid;
+ } else {
+ childNodeIndex = mid;
+ }
+ node = node.getChild(childNodeIndex);
+ return node;
+ }
+
+ /**
+ * Comparison of index key will be following format of key <Dictionary> key
+ * will be in byte array No dictionary key Index of FirstKey (2
+ * bytes)><Index of SecondKey (2 bytes)><Index of NKey (2 bytes)> <First Key
+ * ByteArray><2nd Key ByteArray><N Key ByteArray> in each column value size
+ * of no dictionary column will be -1 if in each column value is not -1 then
+ * compare the byte array based on size and increment the offset to
+ * dictionary column size if size is -1 then its a no dictionary key so to
+ * get the length subtract the size of current with next key offset it will
+ * give the actual length if it is at last position or only one key is
+ * present then subtract with length
+ *
+ * @param first key
+ * @param second key
+ * @return comparison value
+ */
+ private int compareIndexes(IndexKey first, IndexKey second) {
+ int dictionaryKeyOffset = 0;
+ int nonDictionaryKeyOffset = 0;
+ int compareResult = 0;
+ int processedNoDictionaryColumn = numberOfNoDictionaryColumns;
+ ByteBuffer firstNoDictionaryKeyBuffer = ByteBuffer.wrap(first.getNoDictionaryKeys());
+ ByteBuffer secondNoDictionaryKeyBuffer = ByteBuffer.wrap(second.getNoDictionaryKeys());
+ int actualOffset = 0;
+ int firstNoDcitionaryLength = 0;
+ int secondNodeDictionaryLength = 0;
+
+ for (int i = 0; i < eachColumnValueSize.length; i++) {
+
+ if (eachColumnValueSize[i] != NO_DCITIONARY_COLUMN_VALUE) {
+ compareResult = ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(first.getDictionaryKeys(), dictionaryKeyOffset, eachColumnValueSize[i],
+ second.getDictionaryKeys(), dictionaryKeyOffset, eachColumnValueSize[i]);
+ dictionaryKeyOffset += eachColumnValueSize[i];
+ } else {
+ if (processedNoDictionaryColumn > 1) {
+ actualOffset = firstNoDictionaryKeyBuffer.getShort(nonDictionaryKeyOffset);
+ firstNoDcitionaryLength =
+ firstNoDictionaryKeyBuffer.getShort(nonDictionaryKeyOffset + SHORT_SIZE_IN_BYTES);
+ secondNodeDictionaryLength =
+ secondNoDictionaryKeyBuffer.getShort(nonDictionaryKeyOffset + SHORT_SIZE_IN_BYTES);
+ compareResult = ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(first.getNoDictionaryKeys(), actualOffset, firstNoDcitionaryLength,
+ second.getNoDictionaryKeys(), actualOffset, secondNodeDictionaryLength);
+ nonDictionaryKeyOffset += SHORT_SIZE_IN_BYTES;
+ processedNoDictionaryColumn--;
+ } else {
+ actualOffset = firstNoDictionaryKeyBuffer.getShort(nonDictionaryKeyOffset);
+ firstNoDcitionaryLength = first.getNoDictionaryKeys().length - actualOffset;
+ secondNodeDictionaryLength = second.getNoDictionaryKeys().length - actualOffset;
+ compareResult = ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(first.getNoDictionaryKeys(), actualOffset, firstNoDcitionaryLength,
+ second.getNoDictionaryKeys(), actualOffset, secondNodeDictionaryLength);
+ }
+ }
+ if (compareResult != 0) {
+ return compareResult;
+ }
+ }
+
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeNode.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeNode.java
new file mode 100644
index 0000000..6b624c6
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeNode.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.datastore.impl.btree;
+
+import org.apache.carbondata.core.carbon.datastore.DataRefNode;
+import org.apache.carbondata.core.carbon.datastore.IndexKey;
+
+/**
+ * Interface for btree node
+ */
+public interface BTreeNode extends DataRefNode {
+
+ /**
+ * below method will return the one node indexes
+ *
+ * @return node entry array
+ */
+ IndexKey[] getNodeKeys();
+
+ /**
+ * to check whether node in a btree is a leaf node or not
+ *
+ * @return leaf node or not
+ */
+ boolean isLeafNode();
+
+ /**
+ * below method will be used to set the children of intermediate node
+ *
+ * @param children array
+ */
+ void setChildren(BTreeNode[] children);
+
+ /**
+ * below method will used to set the next node
+ *
+ * @param nextNode
+ */
+ void setNextNode(BTreeNode nextNode);
+
+ /**
+ * Below method is to get the children based on index
+ *
+ * @param index children index
+ * @return btree node
+ */
+ BTreeNode getChild(int index);
+
+ /**
+ * below method to set the node entry
+ *
+ * @param key node entry
+ */
+ void setKey(IndexKey key);
+}