You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2016/08/15 07:09:28 UTC
[43/52] [partial] incubator-carbondata git commit: Renamed packages
to org.apache.carbondata and fixed errors
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
new file mode 100644
index 0000000..80a39f1
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
@@ -0,0 +1,425 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.path;
+
+import java.io.File;
+
+import org.apache.carbondata.core.carbon.CarbonTableIdentifier;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
+import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
+import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
+
+import static org.apache.carbondata.core.constants.CarbonCommonConstants.INVALID_SEGMENT_ID;
+
+import org.apache.hadoop.fs.Path;
+
+
+/**
+ * Helps to get Table content paths.
+ */
+public class CarbonTablePath extends Path {
+
+ protected static final String METADATA_DIR = "Metadata";
+ protected static final String DICTIONARY_EXT = ".dict";
+ protected static final String DICTIONARY_META_EXT = ".dictmeta";
+ protected static final String SORT_INDEX_EXT = ".sortindex";
+ protected static final String SCHEMA_FILE = "schema";
+ protected static final String TABLE_STATUS_FILE = "tablestatus";
+ protected static final String FACT_DIR = "Fact";
+ protected static final String AGGREGATE_TABLE_PREFIX = "Agg";
+ protected static final String SEGMENT_PREFIX = "Segment_";
+ protected static final String PARTITION_PREFIX = "Part";
+ protected static final String CARBON_DATA_EXT = ".carbondata";
+ protected static final String DATA_PART_PREFIX = "part";
+ protected static final String INDEX_FILE_EXT = ".carbonindex";
+
+ protected String tablePath;
+ protected CarbonTableIdentifier carbonTableIdentifier;
+
+ /**
+ *
+ * @param carbonTableIdentifier
+ * @param tablePathString
+ */
+ public CarbonTablePath(CarbonTableIdentifier carbonTableIdentifier, String tablePathString) {
+ super(tablePathString);
+ this.carbonTableIdentifier = carbonTableIdentifier;
+ this.tablePath = tablePathString;
+ }
+
+ /**
+ * The method returns the folder path containing the carbon file.
+ *
+ * @param carbonFilePath
+ */
+ public static String getFolderContainingFile(String carbonFilePath) {
+ return carbonFilePath.substring(0, carbonFilePath.lastIndexOf(File.separator));
+ }
+
+ /**
+ * @param columnId unique column identifier
+ * @return name of dictionary file
+ */
+ public static String getDictionaryFileName(String columnId) {
+ return columnId + DICTIONARY_EXT;
+ }
+
+ /**
+ * whether carbonFile is dictionary file or not
+ *
+ * @param carbonFile
+ * @return
+ */
+ public static Boolean isDictionaryFile(CarbonFile carbonFile) {
+ return (!carbonFile.isDirectory()) && (carbonFile.getName().endsWith(DICTIONARY_EXT));
+ }
+
+ /**
+ * check if it is carbon data file matching extension
+ *
+ * @param fileNameWithPath
+ * @return boolean
+ */
+ public static boolean isCarbonDataFile(String fileNameWithPath) {
+ int pos = fileNameWithPath.lastIndexOf('.');
+ if (pos != -1) {
+ return fileNameWithPath.substring(pos).startsWith(CARBON_DATA_EXT);
+ }
+ return false;
+ }
+
+ /**
+ * check if it is carbon index file matching extension
+ *
+ * @param fileNameWithPath
+ * @return boolean
+ */
+ public static boolean isCarbonIndexFile(String fileNameWithPath) {
+ int pos = fileNameWithPath.lastIndexOf('.');
+ if (pos != -1) {
+ return fileNameWithPath.substring(pos).startsWith(INDEX_FILE_EXT);
+ }
+ return false;
+ }
+
+ /**
+ * gets table path
+ */
+ public String getPath() {
+ return tablePath;
+ }
+
+ /**
+ * @param columnId unique column identifier
+ * @return absolute path of dictionary file
+ */
+ public String getDictionaryFilePath(String columnId) {
+ return getMetaDataDir() + File.separator + getDictionaryFileName(columnId);
+ }
+
+ /**
+ * @return it return relative directory
+ */
+ public String getRelativeDictionaryDirectory() {
+ return carbonTableIdentifier.getDatabaseName() + File.separator + carbonTableIdentifier
+ .getTableName();
+ }
+
+ /**
+ * This method will return the metadata directory location for a table
+ *
+ * @return
+ */
+ public String getMetadataDirectoryPath() {
+ return getMetaDataDir();
+ }
+
+ /**
+ * @param columnId unique column identifier
+ * @return absolute path of dictionary meta file
+ */
+ public String getDictionaryMetaFilePath(String columnId) {
+ return getMetaDataDir() + File.separator + columnId + DICTIONARY_META_EXT;
+ }
+
+ /**
+ * @param columnId unique column identifier
+ * @return absolute path of sort index file
+ */
+ public String getSortIndexFilePath(String columnId) {
+ return getMetaDataDir() + File.separator + columnId + SORT_INDEX_EXT;
+ }
+
+ /**
+ *
+ * @param columnId
+ * @param dictOffset
+ * @return absolute path of sortindex with appeneded dictionary offset
+ */
+ public String getSortIndexFilePath(String columnId, long dictOffset) {
+ return getMetaDataDir() + File.separator + columnId + "_" + dictOffset + SORT_INDEX_EXT;
+ }
+
+ /**
+ * @return absolute path of schema file
+ */
+ public String getSchemaFilePath() {
+ return getMetaDataDir() + File.separator + SCHEMA_FILE;
+ }
+
+ /**
+ * @return absolute path of table status file
+ */
+ public String getTableStatusFilePath() {
+ return getMetaDataDir() + File.separator + TABLE_STATUS_FILE;
+ }
+
+ /**
+ * Gets absolute path of data file
+ *
+ * @param partitionId unique partition identifier
+ * @param segmentId unique partition identifier
+ * @param filePartNo data file part number
+ * @param factUpdateTimeStamp unique identifier to identify an update
+ * @return absolute path of data file stored in carbon data format
+ */
+ public String getCarbonDataFilePath(String partitionId, String segmentId, Integer filePartNo,
+ Integer taskNo, String factUpdateTimeStamp) {
+ return getSegmentDir(partitionId, segmentId) + File.separator + getCarbonDataFileName(
+ filePartNo, taskNo, factUpdateTimeStamp);
+ }
+
+ /**
+ * Below method will be used to get the index file present in the segment folder
+ * based on task id
+ *
+ * @param taskId task id of the file
+ * @param partitionId partition number
+ * @param segmentId segment number
+ * @return full qualified carbon index path
+ */
+ public String getCarbonIndexFilePath(final String taskId, final String partitionId,
+ final String segmentId) {
+ String segmentDir = getSegmentDir(partitionId, segmentId);
+ CarbonFile carbonFile =
+ FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir));
+
+ CarbonFile[] files = carbonFile.listFiles(new CarbonFileFilter() {
+ @Override public boolean accept(CarbonFile file) {
+ return file.getName().startsWith(taskId) && file.getName().endsWith(INDEX_FILE_EXT);
+ }
+ });
+ return files[0].getAbsolutePath();
+ }
+
+ /**
+ * Gets absolute path of data file
+ *
+ * @param partitionId unique partition identifier
+ * @param segmentId unique partition identifier
+ * @return absolute path of data file stored in carbon data format
+ */
+ public String getCarbonDataDirectoryPath(String partitionId, String segmentId) {
+ return getSegmentDir(partitionId, segmentId);
+ }
+
+ /**
+ * Gets absolute path of data file of given aggregate table
+ *
+ * @param aggTableID unique aggregate table identifier
+ * @param partitionId unique partition identifier
+ * @param segmentId unique partition identifier
+ * @param filePartNo data file part number
+ * @param factUpdateTimeStamp unique identifier to identify an update
+ * @return absolute path of data file stored in carbon data format
+ */
+ public String getCarbonAggDataFilePath(String aggTableID, String partitionId, String segmentId,
+ Integer filePartNo, Integer taskNo, String factUpdateTimeStamp) {
+ return getAggSegmentDir(aggTableID, partitionId, segmentId) + File.separator
+ + getCarbonDataFileName(filePartNo, taskNo, factUpdateTimeStamp);
+ }
+
+ /**
+ * Gets data file name only with out path
+ *
+ * @param filePartNo data file part number
+ * @param taskNo task identifier
+ * @param factUpdateTimeStamp unique identifier to identify an update
+ * @return gets data file name only with out path
+ */
+ public String getCarbonDataFileName(Integer filePartNo, Integer taskNo,
+ String factUpdateTimeStamp) {
+ return DATA_PART_PREFIX + "-" + filePartNo + "-" + taskNo + "-" + factUpdateTimeStamp
+ + CARBON_DATA_EXT;
+ }
+
+ /**
+ * Below method will be used to get the carbon index filename
+ *
+ * @param taskNo task number
+ * @param factUpdatedTimeStamp time stamp
+ * @return filename
+ */
+ public String getCarbonIndexFileName(int taskNo, String factUpdatedTimeStamp) {
+ return taskNo + "-" + factUpdatedTimeStamp + INDEX_FILE_EXT;
+ }
+
+ private String getSegmentDir(String partitionId, String segmentId) {
+ return getPartitionDir(partitionId) + File.separator + SEGMENT_PREFIX + segmentId;
+ }
+
+ public String getPartitionDir(String partitionId) {
+ return getFactDir() + File.separator + PARTITION_PREFIX + partitionId;
+ }
+
+ private String getAggSegmentDir(String aggTableID, String partitionId, String segmentId) {
+ return getAggPartitionDir(aggTableID, partitionId) + File.separator + SEGMENT_PREFIX
+ + segmentId;
+ }
+
+ private String getAggPartitionDir(String aggTableID, String partitionId) {
+ return getAggregateTableDir(aggTableID) + File.separator + PARTITION_PREFIX + partitionId;
+ }
+
+ private String getMetaDataDir() {
+ return tablePath + File.separator + METADATA_DIR;
+ }
+
+ public String getFactDir() {
+ return tablePath + File.separator + FACT_DIR;
+ }
+
+ private String getAggregateTableDir(String aggTableId) {
+ return tablePath + File.separator + AGGREGATE_TABLE_PREFIX + aggTableId;
+ }
+
+ @Override public boolean equals(Object o) {
+ if (!(o instanceof CarbonTablePath)) {
+ return false;
+ }
+ CarbonTablePath path = (CarbonTablePath) o;
+ return tablePath.equals(path.tablePath) && super.equals(o);
+ }
+
+ @Override public int hashCode() {
+ return super.hashCode() + tablePath.hashCode();
+ }
+
+ /**
+ * To manage data file name and composition
+ */
+ public static class DataFileUtil {
+
+ /**
+ * gets updated timestamp information from given carbon data file name
+ */
+ public static String getUpdateTimeStamp(String carbonDataFileName) {
+ // Get the file name from path
+ String fileName = getFileName(carbonDataFileName);
+ // + 1 for size of "-"
+ int firstDashPos = fileName.indexOf("-");
+ int secondDashPos = fileName.indexOf("-", firstDashPos + 1);
+ int startIndex = fileName.indexOf("-", secondDashPos + 1) + 1;
+ int endIndex = fileName.indexOf(".");
+ return fileName.substring(startIndex, endIndex);
+ }
+
+ /**
+ * gets file part number information from given carbon data file name
+ */
+ public static String getPartNo(String carbonDataFileName) {
+ // Get the file name from path
+ String fileName = getFileName(carbonDataFileName);
+ // + 1 for size of "-"
+ int startIndex = fileName.indexOf("-") + 1;
+ int endIndex = fileName.indexOf("-", startIndex);
+ return fileName.substring(startIndex, endIndex);
+ }
+
+ /**
+ * gets updated timestamp information from given carbon data file name
+ */
+ public static String getTaskNo(String carbonDataFileName) {
+ // Get the file name from path
+ String fileName = getFileName(carbonDataFileName);
+ // + 1 for size of "-"
+ int firstDashPos = fileName.indexOf("-");
+ int startIndex = fileName.indexOf("-", firstDashPos + 1) + 1;
+ int endIndex = fileName.indexOf("-", startIndex);
+ return fileName.substring(startIndex, endIndex);
+ }
+
+ /**
+ * Gets the file name from file path
+ */
+ private static String getFileName(String carbonDataFileName) {
+ int endIndex = carbonDataFileName.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR);
+ if (endIndex > -1) {
+ return carbonDataFileName.substring(endIndex + 1, carbonDataFileName.length());
+ } else {
+ return carbonDataFileName;
+ }
+ }
+ }
+
+ /**
+ * To manage data path and composition
+ */
+ public static class DataPathUtil {
+
+ /**
+ * gets segement id from given absolute data file path
+ */
+ public static String getSegmentId(String dataFileAbsolutePath) {
+ // find segment id from last of data file path
+ int endIndex = dataFileAbsolutePath.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR);
+ // + 1 for size of "/"
+ int startIndex =
+ dataFileAbsolutePath.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR, endIndex - 1) + 1;
+ String segmentDirStr = dataFileAbsolutePath.substring(startIndex, endIndex);
+ //identify id in segment_<id>
+ String[] segmentDirSplits = segmentDirStr.split("_");
+ try {
+ if (segmentDirSplits.length == 2) {
+ return segmentDirSplits[1];
+ }
+ } catch (Exception e) {
+ return INVALID_SEGMENT_ID;
+ }
+ return INVALID_SEGMENT_ID;
+ }
+ }
+
+ /**
+ * Below method will be used to get sort index file present in mentioned folder
+ *
+ * @param sortIndexDir directory where sort index file resides
+ * @param columnUniqueId columnunique id
+ * @return sort index carbon files
+ */
+ public CarbonFile[] getSortIndexFiles(CarbonFile sortIndexDir, final String columnUniqueId) {
+ CarbonFile[] files = sortIndexDir.listFiles(new CarbonFileFilter() {
+ @Override public boolean accept(CarbonFile file) {
+ return file.getName().startsWith(columnUniqueId) && file.getName().endsWith(SORT_INDEX_EXT);
+ }
+ });
+ return files;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatistic.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatistic.java b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatistic.java
new file mode 100644
index 0000000..39198c8
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatistic.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.querystatistics;
+
+import java.io.Serializable;
+
+/**
+ * Wrapper class to maintain the query statistics for each phase of the query
+ */
+public class QueryStatistic implements Serializable {
+
+ /**
+ * serialization id
+ */
+ private static final long serialVersionUID = -5667106646135905848L;
+
+ /**
+ * statistic message
+ */
+ private String message;
+
+ /**
+ * total time take of the phase
+ */
+ private long timeTaken;
+
+ /**
+ * starttime of the phase
+ */
+ private long startTime;
+
+ public QueryStatistic() {
+ this.startTime = System.currentTimeMillis();
+ }
+
+ /**
+ * below method will be used to add the statistic
+ *
+ * @param message Statistic message
+ * @param currentTime current time
+ */
+ public void addStatistics(String message, long currentTime) {
+ this.timeTaken = currentTime - startTime;
+ this.message = message;
+ }
+
+ /**
+ * Below method will be used to add fixed time statistic.
+ * For example total time taken for scan or result preparation
+ *
+ * @param message statistic message
+ * @param timetaken
+ */
+ public void addFixedTimeStatistic(String message, long timetaken) {
+ this.timeTaken = timetaken;
+ this.message = message;
+ }
+
+ /**
+ * Below method will be used to get the statistic message, which will
+ * be used to log
+ *
+ * @param queryWithTaskId query with task id to append in the message
+ * @return statistic message
+ */
+ public String getStatistics(String queryWithTaskId) {
+ return message + " for the taskid : " + queryWithTaskId + " Is : " + timeTaken;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorder.java b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorder.java
new file mode 100644
index 0000000..ce12cae
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsRecorder.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.carbon.querystatistics;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.carbondata.common.logging.LogService;
+import org.apache.carbondata.common.logging.LogServiceFactory;
+
+/**
+ * Class will be used to record and log the query statistics
+ */
+public class QueryStatisticsRecorder implements Serializable {
+
+ private static final LogService LOGGER =
+ LogServiceFactory.getLogService(QueryStatisticsRecorder.class.getName());
+ /**
+ * serialization version
+ */
+ private static final long serialVersionUID = -5719752001674467864L;
+
+ /**
+ * list for statistics to record time taken
+ * by each phase of the query for example aggregation
+ * scanning,block loading time etc.
+ */
+ private List<QueryStatistic> queryStatistics;
+
+ /**
+ * query with taskd
+ */
+ private String queryIWthTask;
+
+ public QueryStatisticsRecorder(String queryId) {
+ queryStatistics = new ArrayList<QueryStatistic>();
+ this.queryIWthTask = queryId;
+ }
+
+ /**
+ * Below method will be used to add the statistics
+ *
+ * @param statistic
+ */
+ public synchronized void recordStatistics(QueryStatistic statistic) {
+ queryStatistics.add(statistic);
+ }
+
+ /**
+ * Below method will be used to log the statistic
+ */
+ public void logStatistics() {
+ for (QueryStatistic statistic : queryStatistics) {
+ LOGGER.statistic(statistic.getStatistics(queryIWthTask));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
new file mode 100644
index 0000000..4fa77ba
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -0,0 +1,892 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.constants;
+
+public final class CarbonCommonConstants {
+ /**
+ * integer size in bytes
+ */
+ public static final int INT_SIZE_IN_BYTE = 4;
+ /**
+ * short size in bytes
+ */
+ public static final int SHORT_SIZE_IN_BYTE = 2;
+ /**
+ * DOUBLE size in bytes
+ */
+ public static final int DOUBLE_SIZE_IN_BYTE = 8;
+ /**
+ * LONG size in bytes
+ */
+ public static final int LONG_SIZE_IN_BYTE = 8;
+ /**
+ * byte to KB conversion factor
+ */
+ public static final int BYTE_TO_KB_CONVERSION_FACTOR = 1024;
+ /**
+ * BYTE_ENCODING
+ */
+ public static final String BYTE_ENCODING = "ISO-8859-1";
+ /**
+ * measure meta data file name
+ */
+ public static final String MEASURE_METADATA_FILE_NAME = "/msrMetaData_";
+ /**
+ * location of the carbon member, hierarchy and fact files
+ */
+ public static final String STORE_LOCATION = "carbon.storelocation";
+ /**
+ * blocklet size in carbon file
+ */
+ public static final String BLOCKLET_SIZE = "carbon.blocklet.size";
+ /**
+ * TODO: max number of blocklets written in a single file?
+ */
+ public static final String MAX_FILE_SIZE = "carbon.max.file.size";
+ /**
+ * Number of cores to be used
+ */
+ public static final String NUM_CORES = "carbon.number.of.cores";
+ /**
+ * carbon sort size
+ */
+ public static final String SORT_SIZE = "carbon.sort.size";
+ /**
+ * default location of the carbon member, hierarchy and fact files
+ */
+ public static final String STORE_LOCATION_DEFAULT_VAL = "../carbon.store";
+ /**
+ * the folder name of kettle home path
+ */
+ public static final String KETTLE_HOME_NAME = "carbonplugins";
+ /**
+ * CARDINALITY_INCREMENT_DEFAULT_VALUE
+ */
+ public static final int CARDINALITY_INCREMENT_VALUE_DEFAULT_VAL = 10;
+ /**
+ * default blocklet size
+ */
+ public static final String BLOCKLET_SIZE_DEFAULT_VAL = "120000";
+ /**
+ * min blocklet size
+ */
+ public static final int BLOCKLET_SIZE_MIN_VAL = 50;
+ /**
+ * max blocklet size
+ */
+ public static final int BLOCKLET_SIZE_MAX_VAL = 12000000;
+ /**
+ * TODO: default value of max number of blocklet written in a single file?
+ */
+ public static final String MAX_FILE_SIZE_DEFAULT_VAL = "1024";
+ /**
+ * TODO: min value of max number of blocklets written in a single file?
+ */
+ public static final int MAX_FILE_SIZE_DEFAULT_VAL_MIN_VAL = 1;
+ /**
+ * max allowed block size for a file. If block size is greater than this value
+ * then the value is reset to default block size for a file
+ */
+ public static final int MAX_FILE_SIZE_DEFAULT_VAL_MAX_VAL = 2048;
+ /**
+ * default value of number of cores to be used
+ */
+ public static final String NUM_CORES_DEFAULT_VAL = "2";
+ /**
+ * min value of number of cores to be used
+ */
+ public static final int NUM_CORES_MIN_VAL = 1;
+ /**
+ * max value of number of cores to be used
+ */
+ public static final int NUM_CORES_MAX_VAL = 32;
+ /**
+ * default carbon sort size
+ */
+ public static final String SORT_SIZE_DEFAULT_VAL = "100000";
+ /**
+ * min carbon sort size
+ */
+ public static final int SORT_SIZE_MIN_VAL = 1000;
+ /**
+ * carbon properties file path
+ */
+ public static final String CARBON_PROPERTIES_FILE_PATH = "../../../conf/carbon.properties";
+ /**
+ * CARBON_DDL_BASE_HDFS_URL
+ */
+ public static final String CARBON_DDL_BASE_HDFS_URL = "carbon.ddl.base.hdfs.url";
+ /**
+ * Slice Meta data file.
+ */
+ public static final String SLICE_METADATA_FILENAME = "sliceMetaData";
+ /**
+ * Load Folder Name
+ */
+ public static final String LOAD_FOLDER = "Segment_";
+ /**
+ * RESTructure Folder
+ */
+ public static final String RESTRUCTRE_FOLDER = "RS_";
+ /**
+ * BYTEBUFFER_SIZE
+ */
+
+ public static final int BYTEBUFFER_SIZE = 24 * 1024;
+ /**
+ * Average constant
+ */
+ public static final String AVERAGE = "avg";
+ /**
+ * Count constant
+ */
+ public static final String COUNT = "count";
+ /**
+ * Count constant
+ */
+ public static final String COUNT_STAR = "countstar";
+ /**
+ * Max constant
+ */
+ public static final String MAX = "max";
+ /**
+ * Min constant
+ */
+ public static final String MIN = "min";
+ /**
+ * distinct count
+ */
+ public static final String DISTINCT_COUNT = "distinct-count";
+ /**
+ * CUSTOM
+ */
+ public static final String CUSTOM = "custom";
+ /**
+ * SUM
+ */
+ public static final String SUM = "sum";
+ /**
+ * DUMMY aggregation function
+ */
+ public static final String DUMMY = "dummy";
+ /**
+ * MEMBER_DEFAULT_VAL
+ */
+ public static final String MEMBER_DEFAULT_VAL = "@NU#LL$!";
+ /**
+ * BLANK_LINE_FLAG
+ */
+ public static final String BLANK_LINE_FLAG = "@NU#LL$!BLANKLINE";
+ /**
+ * FILE STATUS IN-PROGRESS
+ */
+ public static final String FILE_INPROGRESS_STATUS = ".inprogress";
+ /**
+ * CARBON_BADRECORDS_LOCATION
+ */
+ public static final String CARBON_BADRECORDS_LOC = "carbon.badRecords.location";
+ /**
+ * CARBON_BADRECORDS_LOCATION_DEFAULT
+ */
+ public static final String CARBON_BADRECORDS_LOC_DEFAULT_VAL =
+ "../unibi-solutions/system/carbon/badRecords";
+ /**
+ * HIERARCHY_FILE_EXTENSION
+ */
+ public static final String HIERARCHY_FILE_EXTENSION = ".hierarchy";
+ /**
+ * SORT_TEMP_FILE_LOCATION
+ */
+ public static final String SORT_TEMP_FILE_LOCATION = "sortrowtmp";
+ /**
+ * CARBON_RESULT_SIZE_DEFAULT
+ */
+ public static final String LEVEL_FILE_EXTENSION = ".level";
+ /**
+ * FACT_FILE_EXT
+ */
+ public static final String FACT_FILE_EXT = ".carbondata";
+ /**
+ * MEASUREMETADATA_FILE_EXT
+ */
+ public static final String MEASUREMETADATA_FILE_EXT = ".msrmetadata";
+ /**
+ * GRAPH_ROWSET_SIZE
+ */
+ public static final String GRAPH_ROWSET_SIZE = "carbon.graph.rowset.size";
+ /**
+ * GRAPH_ROWSET_SIZE_DEFAULT
+ */
+ public static final String GRAPH_ROWSET_SIZE_DEFAULT = "500";
+ /**
+ * Comment for <code>TYPE_MYSQL</code>
+ */
+ public static final String TYPE_MYSQL = "MYSQL";
+ /**
+ * Comment for <code>TYPE_MSSQL</code>
+ */
+ public static final String TYPE_MSSQL = "MSSQL";
+ /**
+ * Comment for <code>TYPE_ORACLE</code>
+ */
+ public static final String TYPE_ORACLE = "ORACLE";
+ /**
+ * Comment for <code>TYPE_SYBASE</code>
+ */
+ public static final String TYPE_SYBASE = "SYBASE";
+ /**
+ * SORT_INTERMEDIATE_FILES_LIMIT
+ */
+ public static final String SORT_INTERMEDIATE_FILES_LIMIT = "carbon.sort.intermediate.files.limit";
+ /**
+ * SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE
+ */
+ public static final String SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE = "20";
+ /**
+ * MERGERD_EXTENSION
+ */
+ public static final String MERGERD_EXTENSION = ".merge";
+ /**
+ * SORT_FILE_BUFFER_SIZE
+ */
+ public static final String SORT_FILE_BUFFER_SIZE = "carbon.sort.file.buffer.size";
+ /**
+ * no.of records after which counter to be printed
+ */
+ public static final String DATA_LOAD_LOG_COUNTER = "carbon.load.log.counter";
+ /**
+ * DATA_LOAD_LOG_COUNTER_DEFAULT_COUNTER
+ */
+ public static final String DATA_LOAD_LOG_COUNTER_DEFAULT_COUNTER = "500000";
+ /**
+ * SORT_FILE_WRITE_BUFFER_SIZE
+ */
+ public static final String CARBON_SORT_FILE_WRITE_BUFFER_SIZE =
+ "carbon.sort.file.write.buffer.size";
+ /**
+ * SORT_FILE_WRITE_BUFFER_SIZE_DEFAULT_VALUE
+ */
+ public static final String CARBON_SORT_FILE_WRITE_BUFFER_SIZE_DEFAULT_VALUE = "50000";
+ /**
+ * Number of cores to be used while loading
+ */
+ public static final String NUM_CORES_LOADING = "carbon.number.of.cores.while.loading";
+ /**
+ * Number of cores to be used while compacting
+ */
+ public static final String NUM_CORES_COMPACTING = "carbon.number.of.cores.while.compacting";
+ /**
+ * Number of cores to be used for block sort
+ */
+ public static final String NUM_CORES_BLOCK_SORT = "carbon.number.of.cores.block.sort";
+ /**
+ * Default value of number of cores to be used for block sort
+ */
+ public static final String NUM_CORES_BLOCK_SORT_DEFAULT_VAL = "7";
+ /**
+ * Max value of number of cores to be used for block sort
+ */
+ public static final int NUM_CORES_BLOCK_SORT_MAX_VAL = 12;
+ /**
+ * Min value of number of cores to be used for block sort
+ */
+ public static final int NUM_CORES_BLOCK_SORT_MIN_VAL = 1;
+ /**
+ * CSV_READ_BUFFER_SIZE
+ */
+ public static final String CSV_READ_BUFFER_SIZE = "carbon.csv.read.buffersize.byte";
+ /**
+ * CSV_READ_BUFFER_SIZE
+ */
+ public static final String CSV_READ_BUFFER_SIZE_DEFAULT = "50000";
+ /**
+ * CSV_READ_COPIES
+ */
+ public static final String DEFAULT_NUMBER_CORES = "2";
+ /**
+ * CSV_FILE_EXTENSION
+ */
+ public static final String CSV_FILE_EXTENSION = ".csv";
+ /**
+ * COLON_SPC_CHARACTER
+ */
+ public static final String COLON_SPC_CHARACTER = ":!@#COLON#@!:";
+ /**
+ * HASH_SPC_CHARATER
+ */
+ public static final String HASH_SPC_CHARACTER = "#!@:HASH:@!#";
+ /**
+ * SEMICOLON_SPC_CHARATER
+ */
+ public static final String SEMICOLON_SPC_CHARACTER = ";#!@:SEMIC:@!#;";
+ /**
+ * AMPERSAND_SPC_CHARATER
+ */
+ public static final String AMPERSAND_SPC_CHARACTER = "&#!@:AMPER:@!#&";
+ /**
+ * ATTHERATE_SPC_CHARATER
+ */
+ public static final String COMA_SPC_CHARACTER = ",#!:COMA:!#,";
+ /**
+ * HYPHEN_SPC_CHARACTER
+ */
+ public static final String HYPHEN_SPC_CHARACTER = "-#!:HYPHEN:!#-";
+ /**
+ * CARBON_DECIMAL_POINTERS_DEFAULT
+ */
+ public static final byte CARBON_DECIMAL_POINTERS_DEFAULT = 5;
+ /**
+ * SORT_TEMP_FILE_EXT
+ */
+ public static final String SORT_TEMP_FILE_EXT = ".sorttemp";
+ /**
+ * CARBON_MERGE_SORT_READER_THREAD
+ */
+ public static final String CARBON_MERGE_SORT_READER_THREAD = "carbon.merge.sort.reader.thread";
+ /**
+ * CARBON_MERGE_SORT_READER_THREAD_DEFAULTVALUE
+ */
+ public static final String CARBON_MERGE_SORT_READER_THREAD_DEFAULTVALUE = "3";
+ /**
+ * IS_SORT_TEMP_FILE_COMPRESSION_ENABLED
+ */
+ public static final String IS_SORT_TEMP_FILE_COMPRESSION_ENABLED =
+ "carbon.is.sort.temp.file.compression.enabled";
+ /**
+ * IS_SORT_TEMP_FILE_COMPRESSION_ENABLED_DEFAULTVALUE
+ */
+ public static final String IS_SORT_TEMP_FILE_COMPRESSION_ENABLED_DEFAULTVALUE = "false";
+ /**
+ * SORT_TEMP_FILE_NO_OF_RECORDS_FOR_COMPRESSION
+ */
+ public static final String SORT_TEMP_FILE_NO_OF_RECORDS_FOR_COMPRESSION =
+ "carbon.sort.temp.file.no.of.records.for.compression";
+ /**
+ * SORT_TEMP_FILE_NO_OF_RECORD_FOR_COMPRESSION_DEFAULTVALUE
+ */
+ public static final String SORT_TEMP_FILE_NO_OF_RECORD_FOR_COMPRESSION_DEFAULTVALUE = "50";
+ /**
+ * DEFAULT_COLLECTION_SIZE
+ */
+ public static final int DEFAULT_COLLECTION_SIZE = 16;
+ /**
+ * CARBON_TIMESTAMP_DEFAULT_FORMAT
+ */
+ public static final String CARBON_TIMESTAMP_DEFAULT_FORMAT = "yyyy-MM-dd HH:mm:ss";
+ /**
+ * CARBON_TIMESTAMP_DEFAULT_FORMAT
+ */
+ public static final String CARBON_TIMESTAMP_FORMAT = "carbon.timestamp.format";
+ /**
+ * STORE_LOCATION_HDFS
+ */
+ public static final String STORE_LOCATION_HDFS = "carbon.storelocation.hdfs";
+ /**
+ * STORE_LOCATION_TEMP_PATH
+ */
+ public static final String STORE_LOCATION_TEMP_PATH = "carbon.tempstore.location";
+ /**
+ * IS_COLUMNAR_STORAGE_DEFAULTVALUE
+ */
+ public static final String IS_COLUMNAR_STORAGE_DEFAULTVALUE = "true";
+ /**
+ * DIMENSION_SPLIT_VALUE_IN_COLUMNAR_DEFAULTVALUE
+ */
+ public static final String DIMENSION_SPLIT_VALUE_IN_COLUMNAR_DEFAULTVALUE = "1";
+ /**
+ * IS_FULLY_FILLED_BITS_DEFAULT_VALUE
+ */
+ public static final String IS_FULLY_FILLED_BITS_DEFAULT_VALUE = "true";
+ /**
+ * IS_INT_BASED_INDEXER
+ */
+ public static final String AGGREAGATE_COLUMNAR_KEY_BLOCK = "aggregate.columnar.keyblock";
+ /**
+ * IS_INT_BASED_INDEXER_DEFAULTVALUE
+ */
+ public static final String AGGREAGATE_COLUMNAR_KEY_BLOCK_DEFAULTVALUE = "true";
+ /**
+ * TIME_STAT_UTIL_TYPE
+ */
+ public static final String ENABLE_DATA_LOADING_STATISTICS = "enable.data.loading.statistics";
+ /**
+ * TIME_STAT_UTIL_TYPE_DEFAULT
+ */
+ public static final String ENABLE_DATA_LOADING_STATISTICS_DEFAULT = "false";
+ /**
+ * IS_INT_BASED_INDEXER
+ */
+ public static final String HIGH_CARDINALITY_VALUE = "high.cardinality.value";
+ /**
+ * IS_INT_BASED_INDEXER_DEFAULTVALUE
+ */
+ public static final String HIGH_CARDINALITY_VALUE_DEFAULTVALUE = "100000";
+ /**
+ * CONSTANT_SIZE_TEN
+ */
+ public static final int CONSTANT_SIZE_TEN = 10;
+ /**
+ * LEVEL_METADATA_FILE
+ */
+ public static final String LEVEL_METADATA_FILE = "levelmetadata_";
+ public static final String ENABLE_BASE64_ENCODING = "enable.base64.encoding";
+ public static final String ENABLE_BASE64_ENCODING_DEFAULT = "false";
+ /**
+ * LOAD_STATUS SUCCESS
+ */
+ public static final String STORE_LOADSTATUS_SUCCESS = "Success";
+ /**
+ * LOAD_STATUS FAILURE
+ */
+ public static final String STORE_LOADSTATUS_FAILURE = "Failure";
+ /**
+ * LOAD_STATUS PARTIAL_SUCCESS
+ */
+ public static final String STORE_LOADSTATUS_PARTIAL_SUCCESS = "Partial Success";
+ /**
+ * LOAD_STATUS
+ */
+ public static final String CARBON_METADATA_EXTENSION = ".metadata";
+ /**
+ * LOAD_STATUS
+ */
+ public static final String CARBON_DEFAULT_STREAM_ENCODEFORMAT = "UTF-8";
+ /**
+ * AGGREGATE_TABLE_START_TAG
+ */
+ public static final String AGGREGATE_TABLE_START_TAG = "agg";
+ /**
+ * COMMA
+ */
+ public static final String COMMA = ",";
+ /**
+ * UNDERSCORE
+ */
+ public static final String UNDERSCORE = "_";
+ /**
+ * POINT
+ */
+ public static final String POINT = ".";
+ /**
+ * File separator
+ */
+ public static final String FILE_SEPARATOR = "/";
+ /**
+ * MAX_QUERY_EXECUTION_TIME
+ */
+ public static final String MAX_QUERY_EXECUTION_TIME = "max.query.execution.time";
+ /**
+ * CARBON_TIMESTAMP
+ */
+ public static final String CARBON_TIMESTAMP = "dd-MM-yyyy HH:mm:ss";
+ /**
+ * METADATA_LOCK
+ */
+ public static final String METADATA_LOCK = "meta.lock";
+ /**
+ * NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK
+ */
+ public static final int NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK_DEFAULT = 3;
+ /**
+ * MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK
+ */
+ public static final int MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK_DEFAULT = 5;
+ /**
+ * NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK
+ */
+ public static final String NUMBER_OF_TRIES_FOR_LOAD_METADATA_LOCK =
+ "carbon.load.metadata.lock.retries";
+ /**
+ * MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK
+ */
+ public static final String MAX_TIMEOUT_FOR_LOAD_METADATA_LOCK =
+ "carbon.load.metadata.lock.retry.timeout.sec";
+ /**
+ * MARKED_FOR_DELETION
+ */
+ public static final String MARKED_FOR_DELETE = "Marked for Delete";
+ public static final String MARKED_FOR_UPDATE = "Marked for Update";
+ public static final String STRING_TYPE = "StringType";
+ public static final String INTEGER_TYPE = "IntegerType";
+ public static final String LONG_TYPE = "LongType";
+ public static final String DOUBLE_TYPE = "DoubleType";
+ public static final String FLOAT_TYPE = "FloatType";
+ public static final String DATE_TYPE = "DateType";
+ public static final String BOOLEAN_TYPE = "BooleanType";
+ public static final String TIMESTAMP_TYPE = "TimestampType";
+ public static final String BYTE_TYPE = "ByteType";
+ public static final String SHORT_TYPE = "ShortType";
+ public static final String BINARY_TYPE = "BinaryType";
+ public static final String DECIMAL_TYPE = "DecimalType";
+ public static final String STRING = "String";
+ public static final String COLUMNAR = "columnar";
+
+ public static final String INTEGER = "Integer";
+ public static final String SHORT = "Short";
+ public static final String NUMERIC = "Numeric";
+ public static final String TIMESTAMP = "Timestamp";
+ public static final String ARRAY = "ARRAY";
+ public static final String STRUCT = "STRUCT";
+ public static final String INCLUDE = "include";
+ public static final String FROM = "from";
+ public static final String WITH = "with";
+ /**
+ * FACT_UPDATE_EXTENSION.
+ */
+ public static final String FACT_UPDATE_EXTENSION = ".carbondata_update";
+ public static final String FACT_DELETE_EXTENSION = "_delete";
+ /**
+ * MARKED_FOR_UPDATION
+ */
+ public static final String FACT_FILE_UPDATED = "update";
+ /**
+ * MAX_QUERY_EXECUTION_TIME
+ */
+ public static final int DEFAULT_MAX_QUERY_EXECUTION_TIME = 60;
+ /**
+ * LOADMETADATA_FILENAME
+ */
+ public static final String LOADMETADATA_FILENAME = "tablestatus";
+ public static final String SUM_DISTINCT = "sum-distinct";
+ /**
+ * INMEMORY_REOCRD_SIZE
+ */
+ public static final String INMEMORY_REOCRD_SIZE = "carbon.inmemory.record.size";
+ public static final int INMEMORY_REOCRD_SIZE_DEFAULT = 240000;
+
+ /**
+ * INMEMORY_REOCRD_SIZE
+ */
+ public static final String DETAIL_QUERY_BATCH_SIZE = "carbon.detail.batch.size";
+ public static final int DETAIL_QUERY_BATCH_SIZE_DEFAULT = 10000;
+ /**
+ * SPILL_OVER_DISK_PATH
+ */
+ public static final String SCHEMAS_MODIFIED_TIME_FILE = "modifiedTime.mdt";
+ public static final String DEFAULT_INVISIBLE_DUMMY_MEASURE = "default_dummy_measure";
+ /**
+ * max level cache size upto which level cache will be loaded in memory
+ */
+ public static final String CARBON_MAX_LEVEL_CACHE_SIZE = "carbon.max.level.cache.size";
+ /**
+ * max level cache size default value in GB
+ */
+ public static final String CARBON_MAX_LEVEL_CACHE_SIZE_DEFAULT = "-1";
+ /**
+ * DOUBLE_VALUE_MEASURE
+ */
+ public static final char SUM_COUNT_VALUE_MEASURE = 'n';
+ /**
+ * BYTE_VALUE_MEASURE
+ */
+ public static final char BYTE_VALUE_MEASURE = 'c';
+ /**
+ * BIG_DECIMAL_MEASURE
+ */
+ public static final char BIG_DECIMAL_MEASURE = 'b';
+
+ /**
+ * BIG_INT_MEASURE
+ */
+ public static final char BIG_INT_MEASURE = 'l';
+
+ /**
+ * This determines the size of array to be processed in data load steps. one
+ * for dimensions , one of ignore dictionary dimensions , one for measures.
+ */
+ public static final int ARRAYSIZE = 3;
+ /**
+ * CARBON_PREFETCH_BUFFERSIZE
+ */
+ public static final int CARBON_PREFETCH_BUFFERSIZE = 20000;
+ /**
+ * CARBON_PREFETCH_IN_MERGE
+ */
+ public static final boolean CARBON_PREFETCH_IN_MERGE_VALUE = false;
+ /**
+ * TEMPWRITEFILEEXTENSION
+ */
+ public static final String TEMPWRITEFILEEXTENSION = ".write";
+ /**
+ * ENABLE_AUTO_LOAD_MERGE
+ */
+ public static final String ENABLE_AUTO_LOAD_MERGE = "carbon.enable.auto.load.merge";
+ /**
+ * DEFAULT_ENABLE_AUTO_LOAD_MERGE
+ */
+ public static final String DEFAULT_ENABLE_AUTO_LOAD_MERGE = "false";
+
+ /**
+ * ZOOKEEPER_ENABLE_LOCK if this is set to true then zookeeper will be used to handle locking
+ * mechanism of carbon
+ */
+ public static final String LOCK_TYPE = "carbon.lock.type";
+
+ /**
+ * ZOOKEEPER_ENABLE_DEFAULT the default value for zookeeper will be true for carbon
+ */
+ public static final String LOCK_TYPE_DEFAULT = "LOCALLOCK";
+
+ /**
+ * ZOOKEEPER_LOCATION this is the location in zookeeper file system where locks are created.
+ * mechanism of carbon
+ */
+ public static final String ZOOKEEPER_LOCATION = "/CarbonLocks";
+
+ /**
+ * maximum dictionary chunk size that can be kept in memory while writing dictionary file
+ */
+ public static final String DICTIONARY_ONE_CHUNK_SIZE = "carbon.dictionary.chunk.size";
+
+ /**
+ * dictionary chunk default size
+ */
+ public static final String DICTIONARY_ONE_CHUNK_SIZE_DEFAULT = "10000";
+
+ /**
+ * xxhash algorithm property for hashmap
+ */
+ public static final String ENABLE_XXHASH = "carbon.enableXXHash";
+
+ /**
+ * xxhash algorithm property for hashmap. Default value false
+ */
+ public static final String ENABLE_XXHASH_DEFAULT = "true";
+
+ /**
+ * default charset to be used for reading and writing
+ */
+ public static final String DEFAULT_CHARSET = "UTF-8";
+
+ /**
+ * surrogate key that will be sent whenever in the dictionary chunks
+ * a valid surrogate key is not found for a given dictionary value
+ */
+ public static final int INVALID_SURROGATE_KEY = -1;
+
+ /**
+ * surrogate key for MEMBER_DEFAULT_VAL
+ */
+ public static final int MEMBER_DEFAULT_VAL_SURROGATE_KEY = 1;
+
+ public static final String INVALID_SEGMENT_ID = "-1";
+
+ /**
+ * Size of Major Compaction in MBs
+ */
+ public static final String MAJOR_COMPACTION_SIZE = "carbon.major.compaction.size";
+
+ /**
+ * By default size of major compaction in MBs.
+ */
+ public static final String DEFAULT_MAJOR_COMPACTION_SIZE = "1024";
+
+ /**
+ * This property is used to tell how many segments to be preserved from merging.
+ */
+ public static final java.lang.String PRESERVE_LATEST_SEGMENTS_NUMBER =
+ "carbon.numberof.preserve.segments";
+
+ /**
+ * If preserve property is enabled then 2 segments will be preserved.
+ */
+ public static final String DEFAULT_PRESERVE_LATEST_SEGMENTS_NUMBER = "0";
+
+ /**
+ * This property will determine the loads of how many days can be compacted.
+ */
+ public static final java.lang.String DAYS_ALLOWED_TO_COMPACT = "carbon.allowed.compaction.days";
+
+ /**
+ * Default value of 1 day loads can be compacted
+ */
+ public static final String DEFAULT_DAYS_ALLOWED_TO_COMPACT = "0";
+
+ /**
+ * space reserved for writing block meta data in carbon data file
+ */
+ public static final String CARBON_BLOCK_META_RESERVED_SPACE =
+ "carbon.block.meta.size.reserved.percentage";
+
+ /**
+ * default value for space reserved for writing block meta data in carbon data file
+ */
+ public static final String CARBON_BLOCK_META_RESERVED_SPACE_DEFAULT = "10";
+
+ /**
+ * property to enable min max during filter query
+ */
+ public static final String CARBON_QUERY_MIN_MAX_ENABLED = "carbon.enableMinMax";
+
+ /**
+ * default value to enable min or max during filter query execution
+ */
+ public static final String MIN_MAX_DEFAULT_VALUE = "true";
+
+ /**
+ * this variable is to enable/disable prefetch of data during merge sort while
+ * reading data from sort temp files
+ */
+ public static final String CARBON_MERGE_SORT_PREFETCH = "carbon.merge.sort.prefetch";
+ public static final String CARBON_MERGE_SORT_PREFETCH_DEFAULT = "true";
+
+ /**
+ * default name of data base
+ */
+ public static final String DATABASE_DEFAULT_NAME = "default";
+
+ // tblproperties
+ public static final String COLUMN_GROUPS = "column_groups";
+ public static final String DICTIONARY_EXCLUDE = "dictionary_exclude";
+ public static final String DICTIONARY_INCLUDE = "dictionary_include";
+ public static final String PARTITIONCLASS = "partitionclass";
+ public static final String PARTITIONCOUNT = "partitioncount";
+ public static final String COLUMN_PROPERTIES = "columnproperties";
+
+ /**
+ * this variable is to enable/disable identify high cardinality during first data loading
+ */
+ public static final String HIGH_CARDINALITY_IDENTIFY_ENABLE =
+ "high.cardinality.identify.enable";
+ public static final String HIGH_CARDINALITY_IDENTIFY_ENABLE_DEFAULT = "true";
+
+ /**
+ * threshold of high cardinality
+ */
+ public static final String HIGH_CARDINALITY_THRESHOLD = "high.cardinality.threshold";
+ public static final String HIGH_CARDINALITY_THRESHOLD_DEFAULT = "1000000";
+ public static final int HIGH_CARDINALITY_THRESHOLD_MIN = 10000;
+
+ /**
+ * percentage of cardinality in row count
+ */
+ public static final String HIGH_CARDINALITY_IN_ROW_COUNT_PERCENTAGE =
+ "high.cardinality.row.count.percentage";
+ public static final String HIGH_CARDINALITY_IN_ROW_COUNT_PERCENTAGE_DEFAULT = "80";
+
+ /**
+ * 16 mb size
+ */
+ public static final long CARBON_16MB = 16*1024*1024;
+ /**
+ * 256 mb size
+ */
+ public static final long CARBON_256MB = 256*1024*1024;
+
+ /**
+ * Data type String.
+ */
+ public static final String DATATYPE_STRING = "STRING";
+
+ /**
+ * SEGMENT_COMPACTED is property to indicate whether seg is compacted or not.
+ */
+ public static final String SEGMENT_COMPACTED = "Compacted";
+
+ /**
+ * property for number of core to load the blocks in driver
+ */
+ public static final String NUMBER_OF_CORE_TO_LOAD_DRIVER_SEGMENT =
+ "no.of.cores.to.load.blocks.in.driver";
+ /**
+ * default number of cores
+ */
+ public static final int NUMBER_OF_CORE_TO_LOAD_DRIVER_SEGMENT_DEFAULT_VALUE = 10;
+
+ /**
+ * ZOOKEEPERLOCK TYPE
+ */
+ public static final String CARBON_LOCK_TYPE_ZOOKEEPER =
+ "ZOOKEEPERLOCK";
+
+ /**
+ * LOCALLOCK TYPE
+ */
+ public static final String CARBON_LOCK_TYPE_LOCAL =
+ "LOCALLOCK";
+
+ /**
+ * HDFSLOCK TYPE
+ */
+ public static final String CARBON_LOCK_TYPE_HDFS =
+ "HDFSLOCK";
+
+ /**
+ * Lock file in zoo keeper will be of this name.
+ */
+ public static final String ZOOKEEPER_LOCK = "zookeeperLock";
+
+ /**
+ * Invalid filter member log string
+ */
+ public static final String FILTER_INVALID_MEMBER = " Invalid Record(s) are present "
+ + "while filter evaluation. ";
+
+ /**
+ * Number of unmerged segments to be merged.
+ */
+ public static final String COMPACTION_SEGMENT_LEVEL_THRESHOLD =
+ "carbon.compaction.level.threshold";
+
+ /**
+ * Default count for Number of segments to be merged in levels is 4,3
+ */
+ public static final String DEFAULT_SEGMENT_LEVEL_THRESHOLD = "4,3";
+
+ /**
+ * default location of the carbon metastore db
+ */
+ public static final String METASTORE_LOCATION_DEFAULT_VAL = "../carbon.metastore";
+
+ /**
+ * hive connection url
+ */
+ public static final String HIVE_CONNECTION_URL = "javax.jdo.option.ConnectionURL";
+
+ /**
+ * Rocord size in case of compaction.
+ */
+ public static final int COMPACTION_INMEMORY_RECORD_SIZE = 120000;
+
+ /**
+ * If the level 2 compaction is done in minor then new compacted segment will end with .2
+ */
+ public static String LEVEL2_COMPACTION_INDEX = ".2";
+
+ /**
+ * Indicates compaction
+ */
+ public static String COMPACTION_KEY_WORD = "COMPACTION";
+
+ /**
+ * hdfs temporary directory key
+ */
+ public static final String HDFS_TEMP_LOCATION = "hadoop.tmp.dir";
+
+ /**
+ * zookeeper url key
+ */
+ public static final String ZOOKEEPER_URL = "spark.deploy.zookeeper.url";
+
+ private CarbonCommonConstants() {
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/constants/IgnoreDictionary.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/IgnoreDictionary.java b/core/src/main/java/org/apache/carbondata/core/constants/IgnoreDictionary.java
new file mode 100644
index 0000000..6a0105c
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/constants/IgnoreDictionary.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.constants;
+
+/**
+ * This enum is used for determining the indexes of the
+ * dimension,ignoreDictionary,measure columns.
+ */
+public enum IgnoreDictionary {
+ /**
+ * POSITION WHERE DIMENSIONS R STORED IN OBJECT ARRAY.
+ */
+ DIMENSION_INDEX_IN_ROW(0),
+
+ /**
+ * POSITION WHERE BYTE[] (high cardinality) IS STORED IN OBJECT ARRAY.
+ */
+ BYTE_ARRAY_INDEX_IN_ROW(1),
+
+ /**
+ * POSITION WHERE MEASURES R STORED IN OBJECT ARRAY.
+ */
+ MEASURES_INDEX_IN_ROW(2);
+
+ private final int index;
+
+ IgnoreDictionary(int index) {
+ this.index = index;
+ }
+
+ public int getIndex() {
+ return this.index;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/datastorage/store/FileHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/FileHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/FileHolder.java
new file mode 100644
index 0000000..9f8d8ed
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/FileHolder.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.store;
+
+
+public interface FileHolder {
+ /**
+ * This method will be used to read the byte array from file based on offset
+ * and length(number of bytes) need to read
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @param length number of bytes to be read
+ * @return read byte array
+ */
+ byte[] readByteArray(String filePath, long offset, int length);
+
+ /**
+ * This method will be used to read the byte array from file based on length(number of bytes)
+ *
+ * @param filePath fully qualified file path
+ * @param length number of bytes to be read
+ * @return read byte array
+ */
+ byte[] readByteArray(String filePath, int length);
+
+ /**
+ * This method will be used to read int from file from postion(offset), here
+ * length will be always 4 bacause int byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read int
+ */
+ int readInt(String filePath, long offset);
+
+ /**
+ * This method will be used to read long from file from postion(offset), here
+ * length will be always 8 bacause int byte size is 8
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read long
+ */
+ long readLong(String filePath, long offset);
+
+ /**
+ * This method will be used to read int from file from postion(offset), here
+ * length will be always 4 bacause int byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @return read int
+ */
+ int readInt(String filePath);
+
+ /**
+ * This method will be used to read long value from file from postion(offset), here
+ * length will be always 8 because long byte size if 4
+ *
+ * @param filePath fully qualified file path
+ * @param offset reading start position,
+ * @return read long
+ */
+ long readDouble(String filePath, long offset);
+
+ /**
+ * This method will be used to close all the streams currently present in the cache
+ */
+ void finish();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/datastorage/store/MeasureDataWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/MeasureDataWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/MeasureDataWrapper.java
new file mode 100644
index 0000000..80a4374
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/MeasureDataWrapper.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.store;
+
+import org.apache.carbondata.core.datastorage.store.dataholder.CarbonReadDataHolder;
+
+/**
+ * MeasureDataWrapper, interface.
+ */
+public interface MeasureDataWrapper {
+ CarbonReadDataHolder[] getValues();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeKeyStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeKeyStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeKeyStore.java
new file mode 100644
index 0000000..42c5071
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeKeyStore.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.store;
+
+public interface NodeKeyStore {
+ /**
+ * This method will be used to get the actual mdkeys array present in the
+ * store store
+ *
+ * @param fileHolder
+ * @return mdkey
+ */
+ byte[] getBackArray(FileHolder fileHolder);
+
+ /**
+ * This method will be used to insert mdkey to store
+ *
+ * @param index index of mdkey
+ * @param value mdkey
+ */
+ void put(int index, byte[] value);
+
+ /**
+ * This method will be used to get the writable key array.
+ * writable key array will hold below information:
+ * <size of key array><key array>
+ * total length will be 4 bytes for size + key array length
+ *
+ * @return writable array (compressed or normal)
+ */
+ byte[] getWritableKeyArray();
+
+ /**
+ * This method will be used to get the mdkkey array based on index
+ *
+ * @param index index in store
+ * @param fileHolder file holder will be used to read the file
+ * @return mdkey
+ */
+ byte[] get(int index, FileHolder fileHolder);
+
+ /**
+ * This method will clear the store and create the new empty store
+ */
+ void clear();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java
new file mode 100644
index 0000000..f2fe9d2
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/NodeMeasureDataStore.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.store;
+
+import org.apache.carbondata.core.datastorage.store.dataholder.CarbonWriteDataHolder;
+
+public interface NodeMeasureDataStore {
+ /**
+ * This method will be used to get the writable key array.
+ * writable measure data array will hold below information:
+ * <size of measure data array><measure data array>
+ * total length will be 4 bytes for size + measure data array length
+ *
+ * @return writable array (compressed or normal)
+ */
+ byte[][] getWritableMeasureDataArray(CarbonWriteDataHolder[] dataHolderArray);
+
+ MeasureDataWrapper getBackData(int[] cols, FileHolder fileHolder);
+
+ MeasureDataWrapper getBackData(int cols, FileHolder fileHolder);
+
+ short getLength();
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForInt.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForInt.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForInt.java
new file mode 100644
index 0000000..013d873
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForInt.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.store.columnar;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.ByteUtil;
+
+public class BlockIndexerStorageForInt implements IndexStorage<int[]> {
+ private boolean alreadySorted;
+
+ private int[] dataAfterComp;
+
+ private int[] indexMap;
+
+ private byte[][] keyBlock;
+
+ private int[] dataIndexMap;
+
+ private int totalSize;
+
+ public BlockIndexerStorageForInt(byte[][] keyBlock, boolean compressData, boolean isNoDictionary,
+ boolean isSortRequired) {
+ ColumnWithIntIndex[] columnWithIndexs = createColumnWithIndexArray(keyBlock, isNoDictionary);
+ if (isSortRequired) {
+ Arrays.sort(columnWithIndexs);
+ }
+ compressMyOwnWay(extractDataAndReturnIndexes(columnWithIndexs, keyBlock));
+ if (compressData) {
+ compressDataMyOwnWay(columnWithIndexs);
+ }
+ }
+
+ /**
+ * Create an object with each column array and respective index
+ *
+ * @return
+ */
+ private ColumnWithIntIndex[] createColumnWithIndexArray(byte[][] keyBlock,
+ boolean isNoDictionary) {
+ ColumnWithIntIndex[] columnWithIndexs;
+ if (isNoDictionary) {
+ columnWithIndexs = new ColumnWithIntIndexForHighCard[keyBlock.length];
+ for (int i = 0; i < columnWithIndexs.length; i++) {
+ columnWithIndexs[i] = new ColumnWithIntIndexForHighCard(keyBlock[i], i);
+ }
+
+ } else {
+ columnWithIndexs = new ColumnWithIntIndex[keyBlock.length];
+ for (int i = 0; i < columnWithIndexs.length; i++) {
+ columnWithIndexs[i] = new ColumnWithIntIndex(keyBlock[i], i);
+ }
+ }
+
+ return columnWithIndexs;
+ }
+
+ private int[] extractDataAndReturnIndexes(ColumnWithIntIndex[] columnWithIndexs,
+ byte[][] keyBlock) {
+ int[] indexes = new int[columnWithIndexs.length];
+ for (int i = 0; i < indexes.length; i++) {
+ indexes[i] = columnWithIndexs[i].getIndex();
+ keyBlock[i] = columnWithIndexs[i].getColumn();
+ }
+ this.keyBlock = keyBlock;
+ return indexes;
+ }
+
+ /**
+ * It compresses depends up on the sequence numbers.
+ * [1,2,3,4,6,8,10,11,12,13] is translated to [1,4,6,8,10,13] and [0,6]. In
+ * first array the start and end of sequential numbers and second array
+ * keeps the indexes of where sequential numbers starts. If there is no
+ * sequential numbers then the same array it returns with empty second
+ * array.
+ *
+ * @param indexes
+ */
+ public void compressMyOwnWay(int[] indexes) {
+ List<Integer> list = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ int k = 0;
+ int i = 1;
+ for (; i < indexes.length; i++) {
+ if (indexes[i] - indexes[i - 1] == 1) {
+ k++;
+ } else {
+ if (k > 0) {
+ map.add((list.size()));
+ list.add(indexes[i - k - 1]);
+ list.add(indexes[i - 1]);
+ } else {
+ list.add(indexes[i - 1]);
+ }
+ k = 0;
+ }
+ }
+ if (k > 0) {
+ map.add((list.size()));
+ list.add(indexes[i - k - 1]);
+ list.add(indexes[i - 1]);
+ } else {
+ list.add(indexes[i - 1]);
+ }
+ dataAfterComp = convertToArray(list);
+ if (indexes.length == dataAfterComp.length) {
+ indexMap = new int[0];
+ } else {
+ indexMap = convertToArray(map);
+ }
+ if (dataAfterComp.length == 2 && indexMap.length == 1) {
+ alreadySorted = true;
+ }
+ }
+
+ private int[] convertToArray(List<Integer> list) {
+ int[] shortArray = new int[list.size()];
+ for (int i = 0; i < shortArray.length; i++) {
+ shortArray[i] = list.get(i);
+ }
+ return shortArray;
+ }
+
+ /**
+ * @return the alreadySorted
+ */
+ public boolean isAlreadySorted() {
+ return alreadySorted;
+ }
+
+ /**
+ * @return the dataAfterComp
+ */
+ public int[] getDataAfterComp() {
+ return dataAfterComp;
+ }
+
+ /**
+ * @return the indexMap
+ */
+ public int[] getIndexMap() {
+ return indexMap;
+ }
+
+ /**
+ * @return the keyBlock
+ */
+ public byte[][] getKeyBlock() {
+ return keyBlock;
+ }
+
+ private void compressDataMyOwnWay(ColumnWithIntIndex[] indexes) {
+ byte[] prvKey = indexes[0].getColumn();
+ List<ColumnWithIntIndex> list =
+ new ArrayList<ColumnWithIntIndex>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ list.add(indexes[0]);
+ int counter = 1;
+ int start = 0;
+ List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ for (int i = 1; i < indexes.length; i++) {
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, indexes[i].getColumn()) != 0) {
+ prvKey = indexes[i].getColumn();
+ list.add(indexes[i]);
+ map.add(start);
+ map.add(counter);
+ start += counter;
+ counter = 1;
+ continue;
+ }
+ counter++;
+ }
+ map.add(start);
+ map.add(counter);
+ this.keyBlock = convertToKeyArray(list);
+ if (indexes.length == keyBlock.length) {
+ dataIndexMap = new int[0];
+ } else {
+ dataIndexMap = convertToArray(map);
+ }
+ }
+
+ private byte[][] convertToKeyArray(List<ColumnWithIntIndex> list) {
+ byte[][] shortArray = new byte[list.size()][];
+ for (int i = 0; i < shortArray.length; i++) {
+ shortArray[i] = list.get(i).getColumn();
+ totalSize += shortArray[i].length;
+ }
+ return shortArray;
+ }
+
+ @Override public int[] getDataIndexMap() {
+ return dataIndexMap;
+ }
+
+ @Override public int getTotalSize() {
+ return totalSize;
+ }
+
+ @Override public byte[] getMin() {
+ return keyBlock[0];
+ }
+
+ @Override public byte[] getMax() {
+ return keyBlock[keyBlock.length - 1];
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForNoInvertedIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForNoInvertedIndex.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForNoInvertedIndex.java
new file mode 100644
index 0000000..c7d43cf
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/BlockIndexerStorageForNoInvertedIndex.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.store.columnar;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.util.ByteUtil;
+
+public class BlockIndexerStorageForNoInvertedIndex implements IndexStorage<int[]> {
+ private byte[][] keyBlock;
+ private byte[][] sortedBlock;
+ private int totalSize;
+ private int[] dataIndexMap;
+
+ public BlockIndexerStorageForNoInvertedIndex(byte[][] keyBlockInput, boolean compressData,
+ boolean isNoDictionary) {
+ // without invertedindex but can be RLE
+ if (compressData) {
+ // with RLE
+ byte[] prvKey = keyBlockInput[0];
+ List<byte[]> list = new ArrayList<byte[]>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ list.add(keyBlockInput[0]);
+ int counter = 1;
+ int start = 0;
+ List<Integer> map = new ArrayList<Integer>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
+ int length = keyBlockInput.length;
+ for(int i = 1; i < length; i++) {
+ if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(prvKey, keyBlockInput[i]) != 0) {
+ prvKey = keyBlockInput[i];
+ list.add(keyBlockInput[i]);
+ map.add(start);
+ map.add(counter);
+ start += counter;
+ counter = 1;
+ continue;
+ }
+ counter++;
+ }
+ map.add(start);
+ map.add(counter);
+ this.keyBlock = convertToKeyArray(list);
+ if (keyBlockInput.length == this.keyBlock.length) {
+ dataIndexMap = new int[0];
+ } else {
+ dataIndexMap = convertToArray(map);
+ }
+ } else {
+ this.keyBlock = keyBlockInput;
+ dataIndexMap = new int[0];
+ }
+
+ this.sortedBlock = new byte[keyBlock.length][];
+ System.arraycopy(keyBlock, 0, sortedBlock, 0, keyBlock.length);
+ if (isNoDictionary) {
+ Arrays.sort(sortedBlock, new Comparator<byte[]>() {
+ @Override
+ public int compare(byte[] col1, byte[] col2) {
+ return ByteUtil.UnsafeComparer.INSTANCE
+ .compareTo(col1, 2, col1.length - 2, col2, 2, col2.length - 2);
+ }
+ });
+ } else {
+ Arrays.sort(sortedBlock, new Comparator<byte[]>() {
+ @Override
+ public int compare(byte[] col1, byte[] col2) {
+ return ByteUtil.UnsafeComparer.INSTANCE.compareTo(col1, col2);
+ }
+ });
+ }
+
+ }
+
+ private int[] convertToArray(List<Integer> list) {
+ int[] shortArray = new int[list.size()];
+ for(int i = 0; i < shortArray.length; i++) {
+ shortArray[i] = list.get(i);
+ }
+ return shortArray;
+ }
+
+ private byte[][] convertToKeyArray(List<byte[]> list) {
+ byte[][] shortArray = new byte[list.size()][];
+ for (int i = 0; i < shortArray.length; i++) {
+ shortArray[i] = list.get(i);
+ totalSize += shortArray[i].length;
+ }
+ return shortArray;
+ }
+
+ @Override
+ public int[] getDataIndexMap() {
+ return dataIndexMap;
+ }
+
+ @Override
+ public int getTotalSize() {
+ return totalSize;
+ }
+
+ @Override
+ public boolean isAlreadySorted() {
+ return true;
+ }
+
+ /**
+ * no use
+ * @return
+ */
+ @Override
+ public int[] getDataAfterComp() {
+ return new int[0];
+ }
+
+ /**
+ * no use
+ * @return
+ */
+ @Override
+ public int[] getIndexMap() {
+ return new int[0];
+ }
+
+ /**
+ * @return the keyBlock
+ */
+ public byte[][] getKeyBlock() {
+ return keyBlock;
+ }
+
+ @Override public byte[] getMin() {
+ return sortedBlock[0];
+ }
+
+ @Override public byte[] getMax() {
+ return sortedBlock[sortedBlock.length - 1];
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java
new file mode 100644
index 0000000..cf9ba40
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnGroupModel.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.carbondata.core.datastorage.store.columnar;
+
+public class ColumnGroupModel {
+
+ /**
+ * number of columns in columnar block
+ */
+ private int[] columnSplit;
+
+ /**
+ * total number of columns
+ */
+ private int noOfColumnsStore;
+
+ /**
+ * whether given index is columnar or not
+ * true: columnar
+ * false: row block
+ */
+ private boolean[] columnarStore;
+
+ /**
+ * column groups
+ * e.g
+ * {{0,1,2},3,4,{5,6}}
+ */
+ private int[][] columnGroups;
+
+ /**
+ * return columnSplit
+ *
+ * @return
+ */
+ public int[] getColumnSplit() {
+ return columnSplit;
+ }
+
+ /**
+ * set columnSplit
+ *
+ * @param split
+ */
+ public void setColumnSplit(int[] split) {
+ this.columnSplit = split;
+ }
+
+ /**
+ * @return no of columnar block
+ */
+ public int getNoOfColumnStore() {
+ return this.noOfColumnsStore;
+ }
+
+ /**
+ * set no of columnar block
+ *
+ * @param noOfColumnsStore
+ */
+ public void setNoOfColumnStore(int noOfColumnsStore) {
+ this.noOfColumnsStore = noOfColumnsStore;
+ }
+
+ /**
+ * it's an identifier for row block or single column block
+ *
+ * @param columnarStore
+ */
+ public void setColumnarStore(boolean[] columnarStore) {
+ this.columnarStore = columnarStore;
+ }
+
+ /**
+ * set column groups
+ *
+ * @param columnGroups
+ */
+ public void setColumnGroup(int[][] columnGroups) {
+ this.columnGroups = columnGroups;
+ }
+
+ /**
+ * check if given column group is columnar
+ *
+ * @param colGroup
+ * @return true if given block is columnar
+ */
+ public boolean isColumnar(int colGroup) {
+ return columnarStore[colGroup];
+ }
+
+ /**
+ * @return columngroups
+ */
+ public int[][] getColumnGroup() {
+ return this.columnGroups;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndex.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndex.java
new file mode 100644
index 0000000..36606a5
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndex.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.store.columnar;
+
+import java.util.Arrays;
+
+import org.apache.carbondata.core.util.ByteUtil;
+
+public class ColumnWithIntIndex implements Comparable<ColumnWithIntIndex> {
+ protected byte[] column;
+
+ private int index;
+
+ public ColumnWithIntIndex(byte[] column, int index) {
+ this.column = column;
+ this.index = index;
+ }
+
+ public ColumnWithIntIndex() {
+ }
+
+ /**
+ * @return the column
+ */
+ public byte[] getColumn() {
+ return column;
+ }
+
+ /**
+ * @param column the column to set
+ */
+ public void setColumn(byte[] column) {
+ this.column = column;
+ }
+
+ /**
+ * @return the index
+ */
+ public int getIndex() {
+ return index;
+ }
+
+ /**
+ * @param index the index to set
+ */
+ public void setIndex(int index) {
+ this.index = index;
+ }
+
+ @Override public int compareTo(ColumnWithIntIndex o) {
+ return ByteUtil.UnsafeComparer.INSTANCE.compareTo(column, o.column);
+ }
+
+ @Override public boolean equals(Object obj) {
+ if(obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ ColumnWithIntIndex o = (ColumnWithIntIndex)obj;
+ return Arrays.equals(column, o.column) && index == o.index;
+ }
+
+ @Override public int hashCode() {
+ return Arrays.hashCode(column) + index;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndexForHighCard.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndexForHighCard.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndexForHighCard.java
new file mode 100644
index 0000000..61a1165
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnWithIntIndexForHighCard.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.store.columnar;
+
+import java.util.Arrays;
+
+import org.apache.carbondata.core.util.ByteUtil.UnsafeComparer;
+
+public class ColumnWithIntIndexForHighCard extends ColumnWithIntIndex
+ implements Comparable<ColumnWithIntIndex> {
+
+ public ColumnWithIntIndexForHighCard(byte[] column, int index) {
+ super(column, index);
+ }
+
+ @Override public int compareTo(ColumnWithIntIndex o) {
+ return UnsafeComparer.INSTANCE
+ .compareTo(column, 2, column.length - 2, o.column, 2, o.column.length - 2);
+ }
+
+ @Override public boolean equals(Object obj) {
+ if(obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ ColumnWithIntIndexForHighCard o = (ColumnWithIntIndexForHighCard)obj;
+ return Arrays.equals(column, o.column) && getIndex() == o.getIndex();
+ }
+
+ @Override public int hashCode() {
+ return Arrays.hashCode(column) + getIndex();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStore.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStore.java
new file mode 100644
index 0000000..9e12847
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStore.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.store.columnar;
+
+import org.apache.carbondata.core.datastorage.store.FileHolder;
+
+public interface ColumnarKeyStore {
+ /**
+ * This method will be used to get the actual mdkeys array present in the
+ * carbon store, it will read and uncomnpress the key
+ *
+ * @param fileHolder
+ * @return mdkey
+ * @noDictionaryValKeyIndexes, directkey indexes for determining the NO_DICTIONARY
+ * Col inorder to process the direct surrogates data.
+ */
+ ColumnarKeyStoreDataHolder[] getUnCompressedKeyArray(FileHolder fileHolder, int[] blockIndex,
+ boolean[] needCompressedData, int[] noDictionaryValKeyIndexes);
+
+ /**
+ * This method will be used to get the actual mdkeys array present in the
+ * carbon store, it will read and uncomnpress the key
+ *
+ * @param fileHolder
+ * @return mdkey
+ */
+ ColumnarKeyStoreDataHolder getUnCompressedKeyArray(FileHolder fileHolder, int blockIndex,
+ boolean needCompressedData, int[] noDictionaryValKeyIndexes);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolder.java b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolder.java
new file mode 100644
index 0000000..b66d957
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastorage/store/columnar/ColumnarKeyStoreDataHolder.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.carbondata.core.datastorage.store.columnar;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+
+public class ColumnarKeyStoreDataHolder {
+ private byte[] keyblockData;
+ private List<byte[]> noDictionaryValBasedKeyBlockData;
+ private ColumnarKeyStoreMetadata columnarKeyStoreMetadata;
+
+ public ColumnarKeyStoreDataHolder(final byte[] keyblockData,
+ final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) {
+ this.keyblockData = keyblockData;
+ this.columnarKeyStoreMetadata = columnarKeyStoreMetadata;
+ }
+
+ //Added constructor for holding noDictionaryValBasedKeyBlockData
+ public ColumnarKeyStoreDataHolder(final List<byte[]> noDictionaryValBasedKeyBlockData,
+ final ColumnarKeyStoreMetadata columnarKeyStoreMetadata) {
+ this.noDictionaryValBasedKeyBlockData = noDictionaryValBasedKeyBlockData;
+ this.columnarKeyStoreMetadata = columnarKeyStoreMetadata;
+ }
+
+ public byte[] getKeyBlockData() {
+ return keyblockData;
+ }
+
+ /**
+ * @return the columnarKeyStoreMetadata
+ */
+ public ColumnarKeyStoreMetadata getColumnarKeyStoreMetadata() {
+ return columnarKeyStoreMetadata;
+ }
+
+ public void unCompress() {
+ if (columnarKeyStoreMetadata.isUnCompressed()) {
+ return;
+ }
+ this.keyblockData = UnBlockIndexer
+ .uncompressData(keyblockData, columnarKeyStoreMetadata.getDataIndex(),
+ columnarKeyStoreMetadata.getEachRowSize());
+ columnarKeyStoreMetadata.setUnCompressed(true);
+ }
+
+ public int getSurrogateKey(int columnIndex) {
+ byte[] actual = new byte[4];
+ int startIndex;
+ if (null != columnarKeyStoreMetadata.getColumnReverseIndex()) {
+ startIndex =
+ columnarKeyStoreMetadata.getColumnReverseIndex()[columnIndex] * columnarKeyStoreMetadata
+ .getEachRowSize();
+ } else {
+ startIndex = columnIndex * columnarKeyStoreMetadata.getEachRowSize();
+ }
+ int destPos = 4 - columnarKeyStoreMetadata.getEachRowSize();
+ System.arraycopy(keyblockData, startIndex, actual, destPos,
+ columnarKeyStoreMetadata.getEachRowSize());
+ return ByteBuffer.wrap(actual).getInt();
+ }
+
+ /**
+ * get the byte[] for high cardinality column block
+ *
+ * @return List<byte[]>.
+ */
+ public List<byte[]> getNoDictionaryValBasedKeyBlockData() {
+ return noDictionaryValBasedKeyBlockData;
+ }
+
+ /**
+ * set the byte[] for high cardinality column block
+ *
+ * @param noDictionaryValBasedKeyBlockData
+ */
+ public void setNoDictionaryValBasedKeyBlockData(List<byte[]> noDictionaryValBasedKeyBlockData) {
+ this.noDictionaryValBasedKeyBlockData = noDictionaryValBasedKeyBlockData;
+ }
+}