You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2016/08/15 07:08:57 UTC
[12/52] [partial] incubator-carbondata git commit: Renamed packages
to org.apache.carbondata and fixed errors
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/util/CarbonLoadStatisticsImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/util/CarbonLoadStatisticsImpl.java b/core/src/main/java/org/carbondata/core/util/CarbonLoadStatisticsImpl.java
deleted file mode 100644
index 3a56db2..0000000
--- a/core/src/main/java/org/carbondata/core/util/CarbonLoadStatisticsImpl.java
+++ /dev/null
@@ -1,413 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.carbondata.core.util;
-
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.carbondata.common.logging.LogService;
-import org.carbondata.common.logging.LogServiceFactory;
-
-/**
- * A util which provide methods used to record time information druing data loading.
- */
-public class CarbonLoadStatisticsImpl implements LoadStatistics {
- private CarbonLoadStatisticsImpl() {
-
- }
-
- private static CarbonLoadStatisticsImpl carbonLoadStatisticsImplInstance =
- new CarbonLoadStatisticsImpl();
-
- public static CarbonLoadStatisticsImpl getInstance() {
- return carbonLoadStatisticsImplInstance;
- }
-
- private final LogService LOGGER =
- LogServiceFactory.getLogService(CarbonLoadStatisticsImpl.class.getName());
-
- /*
- *We only care about the earliest start time(EST) and the latest end time(LET) of different
- *threads, who does the same thing, LET - EST is the cost time of doing one thing using
- *multiple thread.
- */
- private long loadCsvfilesToDfStartTime = 0;
- private long loadCsvfilesToDfCostTime = 0;
- private long dicShuffleAndWriteFileTotalStartTime = 0;
- private long dicShuffleAndWriteFileTotalCostTime = 0;
-
- //LRU cache load one time
- private double lruCacheLoadTime = 0;
-
- //Generate surrogate keys total time for each partition:
- private ConcurrentHashMap<String, Long[]> parDictionaryValuesTotalTimeMap =
- new ConcurrentHashMap<String, Long[]>();
- private ConcurrentHashMap<String, Long[]> parCsvInputStepTimeMap =
- new ConcurrentHashMap<String, Long[]>();
- private ConcurrentHashMap<String, Long[]> parGeneratingDictionaryValuesTimeMap =
- new ConcurrentHashMap<String, Long[]>();
-
- //Sort rows step total time for each partition:
- private ConcurrentHashMap<String, Long[]> parSortRowsStepTotalTimeMap =
- new ConcurrentHashMap<String, Long[]>();
-
- //MDK generate total time for each partition:
- private ConcurrentHashMap<String, Long[]> parMdkGenerateTotalTimeMap =
- new ConcurrentHashMap<String, Long[]>();
- private ConcurrentHashMap<String, Long[]> parDictionaryValue2MdkAdd2FileTime =
- new ConcurrentHashMap<String, Long[]>();
-
- //Node block process information
- private ConcurrentHashMap<String, Integer> hostBlockMap =
- new ConcurrentHashMap<String, Integer>();
-
- //Partition block process information
- private ConcurrentHashMap<String, Integer> partitionBlockMap =
- new ConcurrentHashMap<String, Integer>();
-
- private long totalRecords = 0;
- private double totalTime = 0;
-
- @Override
- public void initPartitonInfo(String PartitionId) {
- parDictionaryValuesTotalTimeMap.put(PartitionId, new Long[2]);
- parCsvInputStepTimeMap.put(PartitionId, new Long[2]);
- parSortRowsStepTotalTimeMap.put(PartitionId, new Long[2]);
- parGeneratingDictionaryValuesTimeMap.put(PartitionId, new Long[2]);
- parMdkGenerateTotalTimeMap.put(PartitionId, new Long[2]);
- parDictionaryValue2MdkAdd2FileTime.put(PartitionId, new Long[2]);
- }
-
- //Record the time
- public void recordDicShuffleAndWriteTime() {
- Long dicShuffleAndWriteTimePoint = System.currentTimeMillis();
- if (0 == dicShuffleAndWriteFileTotalStartTime) {
- dicShuffleAndWriteFileTotalStartTime = dicShuffleAndWriteTimePoint;
- }
- if (dicShuffleAndWriteTimePoint - dicShuffleAndWriteFileTotalStartTime >
- dicShuffleAndWriteFileTotalCostTime) {
- dicShuffleAndWriteFileTotalCostTime =
- dicShuffleAndWriteTimePoint - dicShuffleAndWriteFileTotalStartTime;
- }
- }
-
- public void recordLoadCsvfilesToDfTime() {
- Long loadCsvfilesToDfTimePoint = System.currentTimeMillis();
- if (0 == loadCsvfilesToDfStartTime) {
- loadCsvfilesToDfStartTime = loadCsvfilesToDfTimePoint;
- }
- if (loadCsvfilesToDfTimePoint - loadCsvfilesToDfStartTime > loadCsvfilesToDfCostTime) {
- loadCsvfilesToDfCostTime = loadCsvfilesToDfTimePoint - loadCsvfilesToDfStartTime;
- }
- }
-
- public double getLruCacheLoadTime() {
- return lruCacheLoadTime;
- }
-
- public void recordDictionaryValuesTotalTime(String partitionID,
- Long dictionaryValuesTotalTimeTimePoint) {
- if (null != parDictionaryValuesTotalTimeMap.get(partitionID)) {
- if (null == parDictionaryValuesTotalTimeMap.get(partitionID)[0]) {
- parDictionaryValuesTotalTimeMap.get(partitionID)[0] = dictionaryValuesTotalTimeTimePoint;
- }
- if (null == parDictionaryValuesTotalTimeMap.get(partitionID)[1] ||
- dictionaryValuesTotalTimeTimePoint - parDictionaryValuesTotalTimeMap.get(partitionID)[0] >
- parDictionaryValuesTotalTimeMap.get(partitionID)[1]) {
- parDictionaryValuesTotalTimeMap.get(partitionID)[1] = dictionaryValuesTotalTimeTimePoint -
- parDictionaryValuesTotalTimeMap.get(partitionID)[0];
- }
- }
- }
-
- public void recordCsvInputStepTime(String partitionID,
- Long csvInputStepTimePoint) {
- if (null != parCsvInputStepTimeMap.get(partitionID)) {
- if (null == parCsvInputStepTimeMap.get(partitionID)[0]) {
- parCsvInputStepTimeMap.get(partitionID)[0] = csvInputStepTimePoint;
- }
- if (null == parCsvInputStepTimeMap.get(partitionID)[1] ||
- csvInputStepTimePoint - parCsvInputStepTimeMap.get(partitionID)[0] >
- parCsvInputStepTimeMap.get(partitionID)[1]) {
- parCsvInputStepTimeMap.get(partitionID)[1] = csvInputStepTimePoint -
- parCsvInputStepTimeMap.get(partitionID)[0];
- }
- }
- }
-
- public void recordLruCacheLoadTime(double lruCacheLoadTime) {
- this.lruCacheLoadTime = lruCacheLoadTime;
- }
-
- public void recordGeneratingDictionaryValuesTime(String partitionID,
- Long generatingDictionaryValuesTimePoint) {
- if (null != parGeneratingDictionaryValuesTimeMap.get(partitionID)) {
- if (null == parGeneratingDictionaryValuesTimeMap.get(partitionID)[0]) {
- parGeneratingDictionaryValuesTimeMap.get(partitionID)[0] =
- generatingDictionaryValuesTimePoint;
- }
- if (null == parGeneratingDictionaryValuesTimeMap.get(partitionID)[1] ||
- generatingDictionaryValuesTimePoint - parGeneratingDictionaryValuesTimeMap
- .get(partitionID)[0] > parGeneratingDictionaryValuesTimeMap
- .get(partitionID)[1]) {
- parGeneratingDictionaryValuesTimeMap.get(partitionID)[1] =
- generatingDictionaryValuesTimePoint - parGeneratingDictionaryValuesTimeMap
- .get(partitionID)[0];
- }
- }
- }
-
- public void recordSortRowsStepTotalTime(String partitionID,
- Long sortRowsStepTotalTimePoint) {
- if (null != parSortRowsStepTotalTimeMap.get(partitionID)) {
- if (null == parSortRowsStepTotalTimeMap.get(partitionID)[0]) {
- parSortRowsStepTotalTimeMap.get(partitionID)[0] = sortRowsStepTotalTimePoint;
- }
- if (null == parSortRowsStepTotalTimeMap.get(partitionID)[1] ||
- sortRowsStepTotalTimePoint - parSortRowsStepTotalTimeMap.get(partitionID)[0] >
- parSortRowsStepTotalTimeMap.get(partitionID)[1]) {
- parSortRowsStepTotalTimeMap.get(partitionID)[1] = sortRowsStepTotalTimePoint -
- parSortRowsStepTotalTimeMap.get(partitionID)[0];
- }
- }
- }
-
- public void recordMdkGenerateTotalTime(String partitionID,
- Long mdkGenerateTotalTimePoint) {
- if (null != parMdkGenerateTotalTimeMap.get(partitionID)) {
- if (null == parMdkGenerateTotalTimeMap.get(partitionID)[0]) {
- parMdkGenerateTotalTimeMap.get(partitionID)[0] = mdkGenerateTotalTimePoint;
- }
- if (null == parMdkGenerateTotalTimeMap.get(partitionID)[1] ||
- mdkGenerateTotalTimePoint - parMdkGenerateTotalTimeMap.get(partitionID)[0] >
- parMdkGenerateTotalTimeMap.get(partitionID)[1]) {
- parMdkGenerateTotalTimeMap.get(partitionID)[1] = mdkGenerateTotalTimePoint -
- parMdkGenerateTotalTimeMap.get(partitionID)[0];
- }
- }
- }
-
- public void recordDictionaryValue2MdkAdd2FileTime(String partitionID,
- Long dictionaryValue2MdkAdd2FileTimePoint) {
- if (null != parDictionaryValue2MdkAdd2FileTime.get(partitionID)) {
- if (null == parDictionaryValue2MdkAdd2FileTime.get(partitionID)[0]) {
- parDictionaryValue2MdkAdd2FileTime.get(partitionID)[0] =
- dictionaryValue2MdkAdd2FileTimePoint;
- }
- if (null == parDictionaryValue2MdkAdd2FileTime.get(partitionID)[1] ||
- dictionaryValue2MdkAdd2FileTimePoint - parDictionaryValue2MdkAdd2FileTime
- .get(partitionID)[0] > parDictionaryValue2MdkAdd2FileTime
- .get(partitionID)[1]) {
- parDictionaryValue2MdkAdd2FileTime.get(partitionID)[1] =
- dictionaryValue2MdkAdd2FileTimePoint - parDictionaryValue2MdkAdd2FileTime
- .get(partitionID)[0];
- }
- }
- }
-
- //Record the node blocks information map
- public void recordHostBlockMap(String host, Integer numBlocks) {
- hostBlockMap.put(host, numBlocks);
- }
-
- //Record the partition blocks information map
- public void recordPartitionBlockMap(String partitionID, Integer numBlocks) {
- partitionBlockMap.put(partitionID, numBlocks);
- }
-
- public void recordTotalRecords(long totalRecords) {
- this.totalRecords = totalRecords;
- }
-
- //Get the time
- private double getDicShuffleAndWriteFileTotalTime() {
- return dicShuffleAndWriteFileTotalCostTime / 1000.0;
- }
-
- private double getLoadCsvfilesToDfTime() {
- return loadCsvfilesToDfCostTime / 1000.0;
- }
-
- private double getDictionaryValuesTotalTime(String partitionID) {
- return parDictionaryValuesTotalTimeMap.get(partitionID)[1] / 1000.0;
- }
-
- private double getCsvInputStepTime(String partitionID) {
- return parCsvInputStepTimeMap.get(partitionID)[1] / 1000.0;
- }
-
- private double getGeneratingDictionaryValuesTime(String partitionID) {
- return parGeneratingDictionaryValuesTimeMap.get(partitionID)[1] / 1000.0;
- }
-
- private double getSortRowsStepTotalTime(String partitionID) {
- return parSortRowsStepTotalTimeMap.get(partitionID)[1] / 1000.0;
- }
-
- private double getDictionaryValue2MdkAdd2FileTime(String partitionID) {
- return parDictionaryValue2MdkAdd2FileTime.get(partitionID)[1] / 1000.0;
- }
-
- //Get the hostBlockMap
- private ConcurrentHashMap<String, Integer> getHostBlockMap() {
- return hostBlockMap;
- }
-
- //Get the partitionBlockMap
- private ConcurrentHashMap<String, Integer> getPartitionBlockMap() {
- return partitionBlockMap;
- }
-
- //Speed calculate
- private long getTotalRecords() {
- return this.totalRecords;
- }
-
- private int getLoadSpeed() {
- return (int)(totalRecords / totalTime);
- }
-
- private int getGenDicSpeed() {
- return (int)(totalRecords / getLoadCsvfilesToDfTime() + getDicShuffleAndWriteFileTotalTime());
- }
-
- private int getReadCSVSpeed(String partitionID) {
- return (int)(totalRecords / getCsvInputStepTime(partitionID));
- }
-
- private int getGenSurKeySpeed(String partitionID) {
- return (int)(totalRecords / getGeneratingDictionaryValuesTime(partitionID));
- }
-
- private int getSortKeySpeed(String partitionID) {
- return (int)(totalRecords / getSortRowsStepTotalTime(partitionID));
- }
-
- private int getMDKSpeed(String partitionID) {
- return (int)(totalRecords / getDictionaryValue2MdkAdd2FileTime(partitionID));
- }
-
- private double getTotalTime(String partitionID) {
- this.totalTime = getLoadCsvfilesToDfTime() + getDicShuffleAndWriteFileTotalTime() +
- getLruCacheLoadTime() + getDictionaryValuesTotalTime(partitionID) +
- getDictionaryValue2MdkAdd2FileTime(partitionID);
- return totalTime;
- }
-
- //Print the statistics information
- private void printDicGenStatisticsInfo() {
- double loadCsvfilesToDfTime = getLoadCsvfilesToDfTime();
- LOGGER.audit("STAGE 1 ->Load csv to DataFrame and generate" +
- " block distinct values: " + loadCsvfilesToDfTime + "(s)");
- double dicShuffleAndWriteFileTotalTime = getDicShuffleAndWriteFileTotalTime();
- LOGGER.audit("STAGE 2 ->Global dict shuffle and write dict file: " +
- + dicShuffleAndWriteFileTotalTime + "(s)");
- }
-
- private void printLruCacheLoadTimeInfo() {
- LOGGER.audit("STAGE 3 ->LRU cache load: " + getLruCacheLoadTime() + "(s)");
- }
-
- private void printDictionaryValuesGenStatisticsInfo(String partitionID) {
- double dictionaryValuesTotalTime = getDictionaryValuesTotalTime(partitionID);
- LOGGER.audit("STAGE 4 ->Total cost of gen dictionary values, sort and write to temp files: "
- + dictionaryValuesTotalTime + "(s)");
- double csvInputStepTime = getCsvInputStepTime(partitionID);
- double generatingDictionaryValuesTime = getGeneratingDictionaryValuesTime(partitionID);
- LOGGER.audit("STAGE 4.1 -> |_read csv file: " + csvInputStepTime + "(s)");
- LOGGER.audit("STAGE 4.2 -> |_transform to surrogate key: "
- + generatingDictionaryValuesTime + "(s)");
- }
-
- private void printSortRowsStepStatisticsInfo(String partitionID) {
- double sortRowsStepTotalTime = getSortRowsStepTotalTime(partitionID);
- LOGGER.audit("STAGE 4.3 -> |_sort rows and write to temp file: "
- + sortRowsStepTotalTime + "(s)");
- }
-
- private void printGenMdkStatisticsInfo(String partitionID) {
- double dictionaryValue2MdkAdd2FileTime = getDictionaryValue2MdkAdd2FileTime(partitionID);
- LOGGER.audit("STAGE 5 ->Transform to MDK, compress and write fact files: "
- + dictionaryValue2MdkAdd2FileTime + "(s)");
- }
-
- //Print the node blocks information
- private void printHostBlockMapInfo() {
- LOGGER.audit("========== BLOCK_INFO ==========");
- if (getHostBlockMap().size() > 0) {
- for (String host: getHostBlockMap().keySet()) {
- LOGGER.audit("BLOCK_INFO ->Node host: " + host);
- LOGGER.audit("BLOCK_INFO ->The block count in this node: " + getHostBlockMap().get(host));
- }
- } else if (getPartitionBlockMap().size() > 0) {
- for (String parID: getPartitionBlockMap().keySet()) {
- LOGGER.audit("BLOCK_INFO ->Partition ID: " + parID);
- LOGGER.audit("BLOCK_INFO ->The block count in this partition: " +
- getPartitionBlockMap().get(parID));
- }
- }
- }
-
- //Print the speed information
- private void printLoadSpeedInfo(String partitionID) {
- LOGGER.audit("===============Load_Speed_Info===============");
- LOGGER.audit("Total Num of Records Processed: " + getTotalRecords());
- LOGGER.audit("Total Time Cost: " + getTotalTime(partitionID) + "(s)");
- LOGGER.audit("Total Load Speed: " + getLoadSpeed() + "records/s");
- LOGGER.audit("Generate Dictionaries Speed: " + getGenDicSpeed() + "records/s");
- LOGGER.audit("Read CSV Speed: " + getReadCSVSpeed(partitionID) + " records/s");
- LOGGER.audit("Generate Surrogate Key Speed: " + getGenSurKeySpeed(partitionID) + " records/s");
- LOGGER.audit("Sort Key/Write Temp Files Speed: " + getSortKeySpeed(partitionID) + " records/s");
- LOGGER.audit("MDK Step Speed: " + getMDKSpeed(partitionID) + " records/s");
- LOGGER.audit("=============================================");
- }
-
- public void printStatisticsInfo(String partitionID) {
- try {
- LOGGER.audit("========== TIME_STATISTICS PartitionID: " + partitionID + "==========");
- printDicGenStatisticsInfo();
- printLruCacheLoadTimeInfo();
- printDictionaryValuesGenStatisticsInfo(partitionID);
- printSortRowsStepStatisticsInfo(partitionID);
- printGenMdkStatisticsInfo(partitionID);
- printHostBlockMapInfo();
- printLoadSpeedInfo(partitionID);
- } catch (Exception e) {
- LOGGER.audit("Can't print Statistics Information");
- } finally {
- resetLoadStatistics();
- }
- }
-
- //Reset the load statistics values
- private void resetLoadStatistics() {
- loadCsvfilesToDfStartTime = 0;
- loadCsvfilesToDfCostTime = 0;
- dicShuffleAndWriteFileTotalStartTime = 0;
- dicShuffleAndWriteFileTotalCostTime = 0;
- lruCacheLoadTime = 0;
- totalRecords = 0;
- totalTime = 0;
- parDictionaryValuesTotalTimeMap.clear();
- parCsvInputStepTimeMap.clear();
- parSortRowsStepTotalTimeMap.clear();
- parGeneratingDictionaryValuesTimeMap.clear();
- parMdkGenerateTotalTimeMap.clear();
- parDictionaryValue2MdkAdd2FileTime.clear();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/util/CarbonMergerUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/util/CarbonMergerUtil.java b/core/src/main/java/org/carbondata/core/util/CarbonMergerUtil.java
deleted file mode 100644
index e2cdf64..0000000
--- a/core/src/main/java/org/carbondata/core/util/CarbonMergerUtil.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.core.util;
-
-import org.carbondata.common.logging.LogService;
-import org.carbondata.common.logging.LogServiceFactory;
-import org.carbondata.core.constants.CarbonCommonConstants;
-
-/**
- * Util class for merge activities of 2 loads.
- */
-public class CarbonMergerUtil {
-
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CarbonMergerUtil.class.getName());
-
- public static int[] getCardinalityFromLevelMetadata(String path, String tableName) {
- int[] localCardinality = null;
- try {
- localCardinality = CarbonUtil.getCardinalityFromLevelMetadataFile(
- path + '/' + CarbonCommonConstants.LEVEL_METADATA_FILE + tableName + ".metadata");
- } catch (CarbonUtilException e) {
- LOGGER.error("Error occurred :: " + e.getMessage());
- }
-
- return localCardinality;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/util/CarbonMetadataUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/carbondata/core/util/CarbonMetadataUtil.java
deleted file mode 100644
index c20e88a..0000000
--- a/core/src/main/java/org/carbondata/core/util/CarbonMetadataUtil.java
+++ /dev/null
@@ -1,450 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.core.util;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-
-import org.carbondata.common.logging.LogService;
-import org.carbondata.common.logging.LogServiceFactory;
-import org.carbondata.core.carbon.datastore.block.SegmentProperties;
-import org.carbondata.core.carbon.metadata.index.BlockIndexInfo;
-import org.carbondata.core.constants.CarbonCommonConstants;
-import org.carbondata.core.datastorage.store.compression.ValueCompressionModel;
-import org.carbondata.core.metadata.BlockletInfoColumnar;
-import org.carbondata.core.metadata.ValueEncoderMeta;
-import org.carbondata.format.BlockIndex;
-import org.carbondata.format.BlockletBTreeIndex;
-import org.carbondata.format.BlockletIndex;
-import org.carbondata.format.BlockletInfo;
-import org.carbondata.format.BlockletMinMaxIndex;
-import org.carbondata.format.ChunkCompressionMeta;
-import org.carbondata.format.ColumnSchema;
-import org.carbondata.format.CompressionCodec;
-import org.carbondata.format.DataChunk;
-import org.carbondata.format.Encoding;
-import org.carbondata.format.FileFooter;
-import org.carbondata.format.IndexHeader;
-import org.carbondata.format.PresenceMeta;
-import org.carbondata.format.SegmentInfo;
-import org.carbondata.format.SortState;
-
-/**
- * Util class to convert to thrift metdata classes
- */
-public class CarbonMetadataUtil {
-
- /**
- * Attribute for Carbon LOGGER
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CarbonMetadataUtil.class.getName());
-
- /**
- * It converts list of BlockletInfoColumnar to FileFooter thrift objects
- *
- * @param infoList
- * @param numCols
- * @param cardinalities
- * @return FileFooter
- */
- public static FileFooter convertFileFooter(List<BlockletInfoColumnar> infoList, int numCols,
- int[] cardinalities, List<ColumnSchema> columnSchemaList,
- SegmentProperties segmentProperties) throws IOException {
-
- SegmentInfo segmentInfo = new SegmentInfo();
- segmentInfo.setNum_cols(columnSchemaList.size());
- segmentInfo.setColumn_cardinalities(CarbonUtil.convertToIntegerList(cardinalities));
-
- FileFooter footer = new FileFooter();
- footer.setNum_rows(getTotalNumberOfRows(infoList));
- footer.setSegment_info(segmentInfo);
- for (BlockletInfoColumnar info : infoList) {
- footer.addToBlocklet_index_list(getBlockletIndex(info));
- }
- footer.setTable_columns(columnSchemaList);
- for (BlockletInfoColumnar info : infoList) {
- footer.addToBlocklet_info_list(getBlockletInfo(info, columnSchemaList, segmentProperties));
- }
- return footer;
- }
-
- private static BlockletIndex getBlockletIndex(
- org.carbondata.core.carbon.metadata.blocklet.index.BlockletIndex info) {
- BlockletMinMaxIndex blockletMinMaxIndex = new BlockletMinMaxIndex();
-
- for (int i = 0; i < info.getMinMaxIndex().getMaxValues().length; i++) {
- blockletMinMaxIndex.addToMax_values(ByteBuffer.wrap(info.getMinMaxIndex().getMaxValues()[i]));
- blockletMinMaxIndex.addToMin_values(ByteBuffer.wrap(info.getMinMaxIndex().getMinValues()[i]));
- }
- BlockletBTreeIndex blockletBTreeIndex = new BlockletBTreeIndex();
- blockletBTreeIndex.setStart_key(info.getBtreeIndex().getStartKey());
- blockletBTreeIndex.setEnd_key(info.getBtreeIndex().getEndKey());
- BlockletIndex blockletIndex = new BlockletIndex();
- blockletIndex.setMin_max_index(blockletMinMaxIndex);
- blockletIndex.setB_tree_index(blockletBTreeIndex);
- return blockletIndex;
- }
-
- /**
- * Get total number of rows for the file.
- *
- * @param infoList
- * @return
- */
- private static long getTotalNumberOfRows(List<BlockletInfoColumnar> infoList) {
- long numberOfRows = 0;
- for (BlockletInfoColumnar info : infoList) {
- numberOfRows += info.getNumberOfKeys();
- }
- return numberOfRows;
- }
-
- private static BlockletIndex getBlockletIndex(BlockletInfoColumnar info) {
-
- BlockletMinMaxIndex blockletMinMaxIndex = new BlockletMinMaxIndex();
- for (byte[] max : info.getColumnMaxData()) {
- blockletMinMaxIndex.addToMax_values(ByteBuffer.wrap(max));
- }
- for (byte[] min : info.getColumnMinData()) {
- blockletMinMaxIndex.addToMin_values(ByteBuffer.wrap(min));
- }
- BlockletBTreeIndex blockletBTreeIndex = new BlockletBTreeIndex();
- blockletBTreeIndex.setStart_key(info.getStartKey());
- blockletBTreeIndex.setEnd_key(info.getEndKey());
-
- BlockletIndex blockletIndex = new BlockletIndex();
- blockletIndex.setMin_max_index(blockletMinMaxIndex);
- blockletIndex.setB_tree_index(blockletBTreeIndex);
- return blockletIndex;
- }
-
- private static BlockletInfo getBlockletInfo(BlockletInfoColumnar blockletInfoColumnar,
- List<ColumnSchema> columnSchenma,
- SegmentProperties segmentProperties) throws IOException {
-
- BlockletInfo blockletInfo = new BlockletInfo();
- blockletInfo.setNum_rows(blockletInfoColumnar.getNumberOfKeys());
-
- List<DataChunk> colDataChunks = new ArrayList<DataChunk>();
- blockletInfoColumnar.getKeyLengths();
- int j = 0;
- int aggregateIndex = 0;
- boolean[] isSortedKeyColumn = blockletInfoColumnar.getIsSortedKeyColumn();
- boolean[] aggKeyBlock = blockletInfoColumnar.getAggKeyBlock();
- boolean[] colGrpblock = blockletInfoColumnar.getColGrpBlocks();
- for (int i = 0; i < blockletInfoColumnar.getKeyLengths().length; i++) {
- DataChunk dataChunk = new DataChunk();
- dataChunk.setChunk_meta(getChunkCompressionMeta());
- List<Encoding> encodings = new ArrayList<Encoding>();
- if (containsEncoding(i, Encoding.DICTIONARY, columnSchenma, segmentProperties)) {
- encodings.add(Encoding.DICTIONARY);
- }
- if (containsEncoding(i, Encoding.DIRECT_DICTIONARY, columnSchenma, segmentProperties)) {
- encodings.add(Encoding.DIRECT_DICTIONARY);
- }
- dataChunk.setRowMajor(colGrpblock[i]);
- //TODO : Once schema PR is merged and information needs to be passed here.
- dataChunk.setColumn_ids(new ArrayList<Integer>());
- dataChunk.setData_page_length(blockletInfoColumnar.getKeyLengths()[i]);
- dataChunk.setData_page_offset(blockletInfoColumnar.getKeyOffSets()[i]);
- if (aggKeyBlock[i]) {
- dataChunk.setRle_page_offset(blockletInfoColumnar.getDataIndexMapOffsets()[aggregateIndex]);
- dataChunk.setRle_page_length(blockletInfoColumnar.getDataIndexMapLength()[aggregateIndex]);
- encodings.add(Encoding.RLE);
- aggregateIndex++;
- }
- dataChunk
- .setSort_state(isSortedKeyColumn[i] ? SortState.SORT_EXPLICIT : SortState.SORT_NATIVE);
-
- if (!isSortedKeyColumn[i]) {
- dataChunk.setRowid_page_offset(blockletInfoColumnar.getKeyBlockIndexOffSets()[j]);
- dataChunk.setRowid_page_length(blockletInfoColumnar.getKeyBlockIndexLength()[j]);
- encodings.add(Encoding.INVERTED_INDEX);
- j++;
- }
-
- //TODO : Right now the encodings are happening at runtime. change as per this encoders.
- dataChunk.setEncoders(encodings);
-
- colDataChunks.add(dataChunk);
- }
-
- for (int i = 0; i < blockletInfoColumnar.getMeasureLength().length; i++) {
- DataChunk dataChunk = new DataChunk();
- dataChunk.setChunk_meta(getChunkCompressionMeta());
- dataChunk.setRowMajor(false);
- //TODO : Once schema PR is merged and information needs to be passed here.
- dataChunk.setColumn_ids(new ArrayList<Integer>());
- dataChunk.setData_page_length(blockletInfoColumnar.getMeasureLength()[i]);
- dataChunk.setData_page_offset(blockletInfoColumnar.getMeasureOffset()[i]);
- //TODO : Right now the encodings are happening at runtime. change as per this encoders.
- List<Encoding> encodings = new ArrayList<Encoding>();
- encodings.add(Encoding.DELTA);
- dataChunk.setEncoders(encodings);
- //TODO writing dummy presence meta need to set actual presence
- //meta
- PresenceMeta presenceMeta = new PresenceMeta();
- presenceMeta.setPresent_bit_streamIsSet(true);
- presenceMeta
- .setPresent_bit_stream(blockletInfoColumnar.getMeasureNullValueIndex()[i].toByteArray());
- dataChunk.setPresence(presenceMeta);
- //TODO : PresenceMeta needs to be implemented and set here
- // dataChunk.setPresence(new PresenceMeta());
- //TODO : Need to write ValueCompression meta here.
- List<ByteBuffer> encoderMetaList = new ArrayList<ByteBuffer>();
- encoderMetaList.add(ByteBuffer.wrap(serializeEncoderMeta(
- createValueEncoderMeta(blockletInfoColumnar.getCompressionModel(), i))));
- dataChunk.setEncoder_meta(encoderMetaList);
- colDataChunks.add(dataChunk);
- }
- blockletInfo.setColumn_data_chunks(colDataChunks);
-
- return blockletInfo;
- }
-
- /**
- * @param blockIndex
- * @param encoding
- * @param columnSchemas
- * @param segmentProperties
- * @return return true if given encoding is present in column
- */
- private static boolean containsEncoding(int blockIndex, Encoding encoding,
- List<ColumnSchema> columnSchemas, SegmentProperties segmentProperties) {
- Set<Integer> dimOrdinals = segmentProperties.getDimensionOrdinalForBlock(blockIndex);
- //column groups will always have dictionary encoding
- if (dimOrdinals.size() > 1 && Encoding.DICTIONARY == encoding) {
- return true;
- }
- for (Integer dimOrdinal : dimOrdinals) {
- if (columnSchemas.get(dimOrdinal).encoders.contains(encoding)) {
- return true;
- }
- }
- return false;
- }
-
- private static byte[] serializeEncoderMeta(ValueEncoderMeta encoderMeta) throws IOException {
- // TODO : should remove the unnecessary fields.
- ByteArrayOutputStream aos = new ByteArrayOutputStream();
- ObjectOutputStream objStream = new ObjectOutputStream(aos);
- objStream.writeObject(encoderMeta);
- objStream.close();
- return aos.toByteArray();
- }
-
- private static ValueEncoderMeta createValueEncoderMeta(ValueCompressionModel compressionModel,
- int index) {
- ValueEncoderMeta encoderMeta = new ValueEncoderMeta();
- encoderMeta.setMaxValue(compressionModel.getMaxValue()[index]);
- encoderMeta.setMinValue(compressionModel.getMinValue()[index]);
- encoderMeta.setDataTypeSelected(compressionModel.getDataTypeSelected()[index]);
- encoderMeta.setDecimal(compressionModel.getDecimal()[index]);
- encoderMeta.setType(compressionModel.getType()[index]);
- encoderMeta.setUniqueValue(compressionModel.getUniqueValue()[index]);
- return encoderMeta;
- }
-
- /**
- * Right now it is set to default values. We may use this in future
- */
- private static ChunkCompressionMeta getChunkCompressionMeta() {
- ChunkCompressionMeta chunkCompressionMeta = new ChunkCompressionMeta();
- chunkCompressionMeta.setCompression_codec(CompressionCodec.SNAPPY);
- chunkCompressionMeta.setTotal_compressed_size(0);
- chunkCompressionMeta.setTotal_uncompressed_size(0);
- return chunkCompressionMeta;
- }
-
- /**
- * It converts FileFooter thrift object to list of BlockletInfoColumnar objects
- *
- * @param footer
- * @return
- */
- public static List<BlockletInfoColumnar> convertBlockletInfo(FileFooter footer)
- throws IOException {
- List<BlockletInfoColumnar> listOfNodeInfo =
- new ArrayList<BlockletInfoColumnar>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
- for (BlockletInfo blockletInfo : footer.getBlocklet_info_list()) {
- BlockletInfoColumnar blockletInfoColumnar = new BlockletInfoColumnar();
- blockletInfoColumnar.setNumberOfKeys(blockletInfo.getNum_rows());
- List<DataChunk> columnChunks = blockletInfo.getColumn_data_chunks();
- List<DataChunk> dictChunks = new ArrayList<DataChunk>();
- List<DataChunk> nonDictColChunks = new ArrayList<DataChunk>();
- for (DataChunk dataChunk : columnChunks) {
- if (dataChunk.getEncoders().get(0).equals(Encoding.DICTIONARY)) {
- dictChunks.add(dataChunk);
- } else {
- nonDictColChunks.add(dataChunk);
- }
- }
- int[] keyLengths = new int[dictChunks.size()];
- long[] keyOffSets = new long[dictChunks.size()];
- long[] keyBlockIndexOffsets = new long[dictChunks.size()];
- int[] keyBlockIndexLens = new int[dictChunks.size()];
- long[] indexMapOffsets = new long[dictChunks.size()];
- int[] indexMapLens = new int[dictChunks.size()];
- boolean[] sortState = new boolean[dictChunks.size()];
- int i = 0;
- for (DataChunk dataChunk : dictChunks) {
- keyLengths[i] = dataChunk.getData_page_length();
- keyOffSets[i] = dataChunk.getData_page_offset();
- keyBlockIndexOffsets[i] = dataChunk.getRowid_page_offset();
- keyBlockIndexLens[i] = dataChunk.getRowid_page_length();
- indexMapOffsets[i] = dataChunk.getRle_page_offset();
- indexMapLens[i] = dataChunk.getRle_page_length();
- sortState[i] = dataChunk.getSort_state().equals(SortState.SORT_EXPLICIT) ? true : false;
- i++;
- }
- blockletInfoColumnar.setKeyLengths(keyLengths);
- blockletInfoColumnar.setKeyOffSets(keyOffSets);
- blockletInfoColumnar.setKeyBlockIndexOffSets(keyBlockIndexOffsets);
- blockletInfoColumnar.setKeyBlockIndexLength(keyBlockIndexLens);
- blockletInfoColumnar.setDataIndexMapOffsets(indexMapOffsets);
- blockletInfoColumnar.setDataIndexMapLength(indexMapLens);
- blockletInfoColumnar.setIsSortedKeyColumn(sortState);
-
- int[] msrLens = new int[nonDictColChunks.size()];
- long[] msrOffsets = new long[nonDictColChunks.size()];
- ValueEncoderMeta[] encoderMetas = new ValueEncoderMeta[nonDictColChunks.size()];
- i = 0;
- for (DataChunk msrChunk : nonDictColChunks) {
- msrLens[i] = msrChunk.getData_page_length();
- msrOffsets[i] = msrChunk.getData_page_offset();
- encoderMetas[i] = deserializeValueEncoderMeta(msrChunk.getEncoder_meta().get(0));
- i++;
- }
- blockletInfoColumnar.setMeasureLength(msrLens);
- blockletInfoColumnar.setMeasureOffset(msrOffsets);
- blockletInfoColumnar.setCompressionModel(getValueCompressionModel(encoderMetas));
- listOfNodeInfo.add(blockletInfoColumnar);
- }
-
- setBlockletIndex(footer, listOfNodeInfo);
- return listOfNodeInfo;
- }
-
- private static ValueEncoderMeta deserializeValueEncoderMeta(ByteBuffer byteBuffer)
- throws IOException {
- ByteArrayInputStream bis = new ByteArrayInputStream(byteBuffer.array());
- ObjectInputStream objStream = new ObjectInputStream(bis);
- ValueEncoderMeta encoderMeta = null;
- try {
- encoderMeta = (ValueEncoderMeta) objStream.readObject();
- } catch (ClassNotFoundException e) {
- LOGGER.error("Error while reading ValueEncoderMeta");
- }
- return encoderMeta;
-
- }
-
- private static ValueCompressionModel getValueCompressionModel(ValueEncoderMeta[] encoderMetas) {
- Object[] maxValue = new Object[encoderMetas.length];
- Object[] minValue = new Object[encoderMetas.length];
- int[] decimalLength = new int[encoderMetas.length];
- Object[] uniqueValue = new Object[encoderMetas.length];
- char[] aggType = new char[encoderMetas.length];
- byte[] dataTypeSelected = new byte[encoderMetas.length];
- for (int i = 0; i < encoderMetas.length; i++) {
- maxValue[i] = encoderMetas[i].getMaxValue();
- minValue[i] = encoderMetas[i].getMinValue();
- decimalLength[i] = encoderMetas[i].getDecimal();
- uniqueValue[i] = encoderMetas[i].getUniqueValue();
- aggType[i] = encoderMetas[i].getType();
- dataTypeSelected[i] = encoderMetas[i].getDataTypeSelected();
- }
- return ValueCompressionUtil
- .getValueCompressionModel(maxValue, minValue, decimalLength, uniqueValue, aggType,
- dataTypeSelected);
- }
-
- private static void setBlockletIndex(FileFooter footer,
- List<BlockletInfoColumnar> listOfNodeInfo) {
- List<BlockletIndex> blockletIndexList = footer.getBlocklet_index_list();
- for (int i = 0; i < blockletIndexList.size(); i++) {
- BlockletBTreeIndex bTreeIndexList = blockletIndexList.get(i).getB_tree_index();
- BlockletMinMaxIndex minMaxIndexList = blockletIndexList.get(i).getMin_max_index();
-
- listOfNodeInfo.get(i).setStartKey(bTreeIndexList.getStart_key());
- listOfNodeInfo.get(i).setEndKey(bTreeIndexList.getEnd_key());
- byte[][] min = new byte[minMaxIndexList.getMin_values().size()][];
- byte[][] max = new byte[minMaxIndexList.getMax_values().size()][];
- for (int j = 0; j < minMaxIndexList.getMax_valuesSize(); j++) {
- min[j] = minMaxIndexList.getMin_values().get(j).array();
- max[j] = minMaxIndexList.getMax_values().get(j).array();
- }
- listOfNodeInfo.get(i).setColumnMaxData(max);
- }
- }
-
- /**
- * Below method will be used to get the index header
- *
- * @param columnCardinality cardinality of each column
- * @param columnSchemaList list of column present in the table
- * @return Index header object
- */
- public static IndexHeader getIndexHeader(int[] columnCardinality,
- List<ColumnSchema> columnSchemaList) {
- // create segment info object
- SegmentInfo segmentInfo = new SegmentInfo();
- // set the number of columns
- segmentInfo.setNum_cols(columnSchemaList.size());
- // setting the column cardinality
- segmentInfo.setColumn_cardinalities(CarbonUtil.convertToIntegerList(columnCardinality));
- // create index header object
- IndexHeader indexHeader = new IndexHeader();
- // set the segment info
- indexHeader.setSegment_info(segmentInfo);
- // set the column names
- indexHeader.setTable_columns(columnSchemaList);
- return indexHeader;
- }
-
- /**
- * Below method will be used to get the block index info thrift object for each block
- * present in the segment
- *
- * @param blockIndexInfoList block index info list
- * @return list of block index
- */
- public static List<BlockIndex> getBlockIndexInfo(List<BlockIndexInfo> blockIndexInfoList) {
- List<BlockIndex> thriftBlockIndexList = new ArrayList<BlockIndex>();
- BlockIndex blockIndex = null;
- // below code to create block index info object for each block
- for (BlockIndexInfo blockIndexInfo : blockIndexInfoList) {
- blockIndex = new BlockIndex();
- blockIndex.setNum_rows(blockIndexInfo.getNumberOfRows());
- blockIndex.setOffset(blockIndexInfo.getNumberOfRows());
- blockIndex.setFile_name(blockIndexInfo.getFileName());
- blockIndex.setBlock_index(getBlockletIndex(blockIndexInfo.getBlockletIndex()));
- thriftBlockIndexList.add(blockIndex);
- }
- return thriftBlockIndexList;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/carbondata/core/util/CarbonProperties.java
deleted file mode 100644
index c6a3a1b..0000000
--- a/core/src/main/java/org/carbondata/core/util/CarbonProperties.java
+++ /dev/null
@@ -1,494 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.core.util;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.Properties;
-
-import org.carbondata.common.logging.LogService;
-import org.carbondata.common.logging.LogServiceFactory;
-import org.carbondata.core.constants.CarbonCommonConstants;
-
-public final class CarbonProperties {
- /**
- * Attribute for Carbon LOGGER.
- */
- private static final LogService LOGGER =
- LogServiceFactory.getLogService(CarbonProperties.class.getName());
-
- /**
- * class instance.
- */
- private static final CarbonProperties CARBONPROPERTIESINSTANCE = new CarbonProperties();
-
- /**
- * porpeties .
- */
- private Properties carbonProperties;
-
- /**
- * Private constructor this will call load properties method to load all the
- * carbon properties in memory.
- */
- private CarbonProperties() {
- carbonProperties = new Properties();
- loadProperties();
- validateAndLoadDefaultProperties();
- }
-
- /**
- * This method will be responsible for get this class instance
- *
- * @return carbon properties instance
- */
- public static CarbonProperties getInstance() {
- return CARBONPROPERTIESINSTANCE;
- }
-
- /**
- * This method validates the loaded properties and loads default
- * values in case of wrong values.
- */
- private void validateAndLoadDefaultProperties() {
- if (null == carbonProperties.getProperty(CarbonCommonConstants.STORE_LOCATION)) {
- carbonProperties.setProperty(CarbonCommonConstants.STORE_LOCATION,
- CarbonCommonConstants.STORE_LOCATION_DEFAULT_VAL);
- }
-
- validateBlockletSize();
- validateMaxFileSize();
- validateNumCores();
- validateNumCoresBlockSort();
- validateSortSize();
- validateBadRecordsLocation();
- validateHighCardinalityIdentify();
- validateHighCardinalityThreshold();
- validateHighCardinalityInRowCountPercentage();
- }
-
- private void validateBadRecordsLocation() {
- String badRecordsLocation =
- carbonProperties.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC);
- if (null == badRecordsLocation || badRecordsLocation.length() == 0) {
- carbonProperties.setProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC,
- CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL);
- }
- }
-
- /**
- * This method validates the blocklet size
- */
- private void validateBlockletSize() {
- String blockletSizeStr = carbonProperties.getProperty(CarbonCommonConstants.BLOCKLET_SIZE,
- CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL);
- try {
- int blockletSize = Integer.parseInt(blockletSizeStr);
-
- if (blockletSize < CarbonCommonConstants.BLOCKLET_SIZE_MIN_VAL
- || blockletSize > CarbonCommonConstants.BLOCKLET_SIZE_MAX_VAL) {
- LOGGER.info("The blocklet size value \"" + blockletSizeStr
- + "\" is invalid. Using the default value \""
- + CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL);
- carbonProperties.setProperty(CarbonCommonConstants.BLOCKLET_SIZE,
- CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL);
- }
- } catch (NumberFormatException e) {
- LOGGER.info("The blocklet size value \"" + blockletSizeStr
- + "\" is invalid. Using the default value \""
- + CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL);
- carbonProperties.setProperty(CarbonCommonConstants.BLOCKLET_SIZE,
- CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL);
- }
- }
-
- /**
- * TODO: This method validates the maximum number of blocklets per file ?
- */
- private void validateMaxFileSize() {
- String maxFileSizeStr = carbonProperties.getProperty(CarbonCommonConstants.MAX_FILE_SIZE,
- CarbonCommonConstants.MAX_FILE_SIZE_DEFAULT_VAL);
- try {
- int maxFileSize = Integer.parseInt(maxFileSizeStr);
-
- if (maxFileSize < CarbonCommonConstants.MAX_FILE_SIZE_DEFAULT_VAL_MIN_VAL
- || maxFileSize > CarbonCommonConstants.MAX_FILE_SIZE_DEFAULT_VAL_MAX_VAL) {
- LOGGER.info("The max file size value \"" + maxFileSizeStr
- + "\" is invalid. Using the default value \""
- + CarbonCommonConstants.MAX_FILE_SIZE_DEFAULT_VAL);
- carbonProperties.setProperty(CarbonCommonConstants.MAX_FILE_SIZE,
- CarbonCommonConstants.MAX_FILE_SIZE_DEFAULT_VAL);
- }
- } catch (NumberFormatException e) {
- LOGGER.info("The max file size value \"" + maxFileSizeStr
- + "\" is invalid. Using the default value \""
- + CarbonCommonConstants.MAX_FILE_SIZE_DEFAULT_VAL);
-
- carbonProperties.setProperty(CarbonCommonConstants.MAX_FILE_SIZE,
- CarbonCommonConstants.MAX_FILE_SIZE_DEFAULT_VAL);
- }
- }
-
- /**
- * This method validates the number cores specified
- */
- private void validateNumCores() {
- String numCoresStr = carbonProperties
- .getProperty(CarbonCommonConstants.NUM_CORES, CarbonCommonConstants.NUM_CORES_DEFAULT_VAL);
- try {
- int numCores = Integer.parseInt(numCoresStr);
-
- if (numCores < CarbonCommonConstants.NUM_CORES_MIN_VAL
- || numCores > CarbonCommonConstants.NUM_CORES_MAX_VAL) {
- LOGGER.info("The num Cores value \"" + numCoresStr
- + "\" is invalid. Using the default value \""
- + CarbonCommonConstants.NUM_CORES_DEFAULT_VAL);
- carbonProperties.setProperty(CarbonCommonConstants.NUM_CORES,
- CarbonCommonConstants.NUM_CORES_DEFAULT_VAL);
- }
- } catch (NumberFormatException e) {
- LOGGER.info("The num Cores value \"" + numCoresStr
- + "\" is invalid. Using the default value \""
- + CarbonCommonConstants.NUM_CORES_DEFAULT_VAL);
- carbonProperties.setProperty(CarbonCommonConstants.NUM_CORES,
- CarbonCommonConstants.NUM_CORES_DEFAULT_VAL);
- }
- }
-
- /**
- * This method validates the number cores specified for mdk block sort
- */
- private void validateNumCoresBlockSort() {
- String numCoresStr = carbonProperties
- .getProperty(CarbonCommonConstants.NUM_CORES_BLOCK_SORT,
- CarbonCommonConstants.NUM_CORES_BLOCK_SORT_DEFAULT_VAL);
- try {
- int numCores = Integer.parseInt(numCoresStr);
-
- if (numCores < CarbonCommonConstants.NUM_CORES_BLOCK_SORT_MIN_VAL
- || numCores > CarbonCommonConstants.NUM_CORES_BLOCK_SORT_MAX_VAL) {
- LOGGER.info("The num cores value \"" + numCoresStr
- + "\" for block sort is invalid. Using the default value \""
- + CarbonCommonConstants.NUM_CORES_BLOCK_SORT_DEFAULT_VAL);
- carbonProperties.setProperty(CarbonCommonConstants.NUM_CORES_BLOCK_SORT,
- CarbonCommonConstants.NUM_CORES_BLOCK_SORT_DEFAULT_VAL);
- }
- } catch (NumberFormatException e) {
- LOGGER.info("The num cores value \"" + numCoresStr
- + "\" for block sort is invalid. Using the default value \""
- + CarbonCommonConstants.NUM_CORES_BLOCK_SORT_DEFAULT_VAL);
- carbonProperties.setProperty(CarbonCommonConstants.NUM_CORES_BLOCK_SORT,
- CarbonCommonConstants.NUM_CORES_BLOCK_SORT_DEFAULT_VAL);
- }
- }
-
- /**
- * This method validates the sort size
- */
- private void validateSortSize() {
- String sortSizeStr = carbonProperties
- .getProperty(CarbonCommonConstants.SORT_SIZE, CarbonCommonConstants.SORT_SIZE_DEFAULT_VAL);
- try {
- int sortSize = Integer.parseInt(sortSizeStr);
-
- if (sortSize < CarbonCommonConstants.SORT_SIZE_MIN_VAL) {
- LOGGER.info("The batch size value \"" + sortSizeStr
- + "\" is invalid. Using the default value \""
- + CarbonCommonConstants.SORT_SIZE_DEFAULT_VAL);
- carbonProperties.setProperty(CarbonCommonConstants.SORT_SIZE,
- CarbonCommonConstants.SORT_SIZE_DEFAULT_VAL);
- }
- } catch (NumberFormatException e) {
- LOGGER.info("The batch size value \"" + sortSizeStr
- + "\" is invalid. Using the default value \""
- + CarbonCommonConstants.SORT_SIZE_DEFAULT_VAL);
- carbonProperties.setProperty(CarbonCommonConstants.SORT_SIZE,
- CarbonCommonConstants.SORT_SIZE_DEFAULT_VAL);
- }
- }
-
- private void validateHighCardinalityIdentify() {
- String highcardIdentifyStr = carbonProperties.getProperty(
- CarbonCommonConstants.HIGH_CARDINALITY_IDENTIFY_ENABLE,
- CarbonCommonConstants.HIGH_CARDINALITY_IDENTIFY_ENABLE_DEFAULT);
- try {
- Boolean.parseBoolean(highcardIdentifyStr);
- } catch (NumberFormatException e) {
- LOGGER.info("The high cardinality identify value \"" + highcardIdentifyStr
- + "\" is invalid. Using the default value \""
- + CarbonCommonConstants.HIGH_CARDINALITY_IDENTIFY_ENABLE_DEFAULT);
- carbonProperties.setProperty(CarbonCommonConstants.HIGH_CARDINALITY_IDENTIFY_ENABLE,
- CarbonCommonConstants.HIGH_CARDINALITY_IDENTIFY_ENABLE_DEFAULT);
- }
- }
-
- private void validateHighCardinalityThreshold() {
- String highcardThresholdStr = carbonProperties.getProperty(
- CarbonCommonConstants.HIGH_CARDINALITY_THRESHOLD,
- CarbonCommonConstants.HIGH_CARDINALITY_THRESHOLD_DEFAULT);
- try {
- int highcardThreshold = Integer.parseInt(highcardThresholdStr);
- if(highcardThreshold < CarbonCommonConstants.HIGH_CARDINALITY_THRESHOLD_MIN){
- LOGGER.info("The high cardinality threshold value \"" + highcardThresholdStr
- + "\" is invalid. Using the min value \""
- + CarbonCommonConstants.HIGH_CARDINALITY_THRESHOLD_MIN);
- carbonProperties.setProperty(CarbonCommonConstants.HIGH_CARDINALITY_THRESHOLD,
- CarbonCommonConstants.HIGH_CARDINALITY_THRESHOLD_MIN + "");
- }
- } catch (NumberFormatException e) {
- LOGGER.info("The high cardinality threshold value \"" + highcardThresholdStr
- + "\" is invalid. Using the default value \""
- + CarbonCommonConstants.HIGH_CARDINALITY_THRESHOLD_DEFAULT);
- carbonProperties.setProperty(CarbonCommonConstants.HIGH_CARDINALITY_THRESHOLD,
- CarbonCommonConstants.HIGH_CARDINALITY_THRESHOLD_DEFAULT);
- }
- }
-
- private void validateHighCardinalityInRowCountPercentage() {
- String highcardPercentageStr = carbonProperties.getProperty(
- CarbonCommonConstants.HIGH_CARDINALITY_IN_ROW_COUNT_PERCENTAGE,
- CarbonCommonConstants.HIGH_CARDINALITY_IN_ROW_COUNT_PERCENTAGE_DEFAULT);
- try {
- double highcardPercentage = Double.parseDouble(highcardPercentageStr);
- if(highcardPercentage <= 0){
- LOGGER.info("The percentage of high cardinality in row count value \""
- + highcardPercentageStr + "\" is invalid. Using the default value \""
- + CarbonCommonConstants.HIGH_CARDINALITY_IN_ROW_COUNT_PERCENTAGE_DEFAULT);
- carbonProperties.setProperty(
- CarbonCommonConstants.HIGH_CARDINALITY_IN_ROW_COUNT_PERCENTAGE,
- CarbonCommonConstants.HIGH_CARDINALITY_IN_ROW_COUNT_PERCENTAGE_DEFAULT);
- }
- } catch (NumberFormatException e) {
- LOGGER.info("The percentage of high cardinality in row count value \""
- + highcardPercentageStr + "\" is invalid. Using the default value \""
- + CarbonCommonConstants.HIGH_CARDINALITY_IN_ROW_COUNT_PERCENTAGE_DEFAULT);
- carbonProperties.setProperty(CarbonCommonConstants.HIGH_CARDINALITY_IN_ROW_COUNT_PERCENTAGE,
- CarbonCommonConstants.HIGH_CARDINALITY_IN_ROW_COUNT_PERCENTAGE_DEFAULT);
- }
- }
-
- /**
- * This method will read all the properties from file and load it into
- * memory
- */
- private void loadProperties() {
- String property = System.getProperty("carbon.properties.filepath");
- if (null == property) {
- property = CarbonCommonConstants.CARBON_PROPERTIES_FILE_PATH;
- }
- File file = new File(property);
- LOGGER.info("Property file path: " + file.getAbsolutePath());
-
- FileInputStream fis = null;
- try {
- if (file.exists()) {
- fis = new FileInputStream(file);
-
- carbonProperties.load(fis);
- }
- } catch (FileNotFoundException e) {
- LOGGER.error("The file: " + CarbonCommonConstants.CARBON_PROPERTIES_FILE_PATH
- + " does not exist");
- } catch (IOException e) {
- LOGGER.error("Error while reading the file: "
- + CarbonCommonConstants.CARBON_PROPERTIES_FILE_PATH);
- } finally {
- if (null != fis) {
- try {
- fis.close();
- } catch (IOException e) {
- LOGGER.error("Error while closing the file stream for file: "
- + CarbonCommonConstants.CARBON_PROPERTIES_FILE_PATH);
- }
- }
- }
-
- print();
- }
-
- /**
- * This method will be used to get the properties value
- *
- * @param key
- * @return properties value
- */
- public String getProperty(String key) {
- //TODO temporary fix
- if ("carbon.leaf.node.size".equals(key)) {
- return "120000";
- }
- return carbonProperties.getProperty(key);
- }
-
- /**
- * This method will be used to get the properties value if property is not
- * present then it will return tghe default value
- *
- * @param key
- * @return properties value
- */
- public String getProperty(String key, String defaultValue) {
- String value = getProperty(key);
- if (null == value) {
- return defaultValue;
- }
- return value;
- }
-
- /**
- * This method will be used to add a new property
- *
- * @param key
- * @return properties value
- */
- public void addProperty(String key, String value) {
- carbonProperties.setProperty(key, value);
-
- }
-
- /**
- * Validate the restrictions
- *
- * @param actual
- * @param max
- * @param min
- * @param defaultVal
- * @return
- */
- public long validate(long actual, long max, long min, long defaultVal) {
- if (actual <= max && actual >= min) {
- return actual;
- }
- return defaultVal;
- }
-
- /**
- * returns major compaction size value from carbon properties or default value if it is not valid
- *
- * @return
- */
- public long getMajorCompactionSize() {
- long compactionSize;
- try {
- compactionSize = Long.parseLong(getProperty(CarbonCommonConstants.MAJOR_COMPACTION_SIZE,
- CarbonCommonConstants.DEFAULT_MAJOR_COMPACTION_SIZE));
- } catch (NumberFormatException e) {
- compactionSize = Long.parseLong(CarbonCommonConstants.DEFAULT_MAJOR_COMPACTION_SIZE);
- }
- return compactionSize;
- }
-
- /**
- * returns the number of loads to be preserved.
- *
- * @return
- */
- public int getNumberOfSegmentsToBePreserved() {
- int numberOfSegmentsToBePreserved;
- try {
- numberOfSegmentsToBePreserved = Integer.parseInt(
- getProperty(CarbonCommonConstants.PRESERVE_LATEST_SEGMENTS_NUMBER,
- CarbonCommonConstants.DEFAULT_PRESERVE_LATEST_SEGMENTS_NUMBER));
- // checking min and max . 0 , 100 is min & max.
- if (numberOfSegmentsToBePreserved < 0 || numberOfSegmentsToBePreserved > 100) {
- LOGGER.error("The specified value for property "
- + CarbonCommonConstants.PRESERVE_LATEST_SEGMENTS_NUMBER + " is incorrect."
- + " Correct value should be in range of 0 -100. Taking the default value.");
- numberOfSegmentsToBePreserved =
- Integer.parseInt(CarbonCommonConstants.DEFAULT_PRESERVE_LATEST_SEGMENTS_NUMBER);
- }
- } catch (NumberFormatException e) {
- numberOfSegmentsToBePreserved =
- Integer.parseInt(CarbonCommonConstants.DEFAULT_PRESERVE_LATEST_SEGMENTS_NUMBER);
- }
- return numberOfSegmentsToBePreserved;
- }
-
- public void print() {
- LOGGER.info("------Using Carbon.properties --------");
- LOGGER.info(carbonProperties.toString());
- }
-
- /**
- * gettting the unmerged segment numbers to be merged.
- * @return
- */
- public int[] getCompactionSegmentLevelCount() {
- String commaSeparatedLevels;
-
- commaSeparatedLevels = getProperty(CarbonCommonConstants.COMPACTION_SEGMENT_LEVEL_THRESHOLD,
- CarbonCommonConstants.DEFAULT_SEGMENT_LEVEL_THRESHOLD);
- int[] compactionSize = getIntArray(commaSeparatedLevels);
-
- if(null == compactionSize){
- compactionSize = getIntArray(CarbonCommonConstants.DEFAULT_SEGMENT_LEVEL_THRESHOLD);
- }
-
- return compactionSize;
- }
-
- /**
- *
- * @param commaSeparatedLevels
- * @return
- */
- private int[] getIntArray(String commaSeparatedLevels) {
- String[] levels = commaSeparatedLevels.split(",");
- int[] compactionSize = new int[levels.length];
- int i = 0;
- for (String levelSize : levels) {
- try {
- int size = Integer.parseInt(levelSize.trim());
- if(validate(size,100,0,-1) < 0 ){
- // if given size is out of boundary then take default value for all levels.
- return null;
- }
- compactionSize[i++] = size;
- }
- catch(NumberFormatException e){
- LOGGER.error(
- "Given value for property" + CarbonCommonConstants.COMPACTION_SEGMENT_LEVEL_THRESHOLD
- + " is not proper. Taking the default value "
- + CarbonCommonConstants.DEFAULT_SEGMENT_LEVEL_THRESHOLD);
- return null;
- }
- }
- return compactionSize;
- }
-
- /**
- * Validate the restrictions
- *
- * @param actual
- * @param max
- * @param min
- * @param defaultVal
- * @return
- */
- public int validate(int actual, int max, int min, int defaultVal) {
- if (actual <= max && actual >= min) {
- return actual;
- }
- return defaultVal;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/util/CarbonTimeStatisticsFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/util/CarbonTimeStatisticsFactory.java b/core/src/main/java/org/carbondata/core/util/CarbonTimeStatisticsFactory.java
deleted file mode 100644
index a5c67ff..0000000
--- a/core/src/main/java/org/carbondata/core/util/CarbonTimeStatisticsFactory.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.carbondata.core.util;
-
-import org.carbondata.core.constants.CarbonCommonConstants;
-
-public class CarbonTimeStatisticsFactory {
- private static String LoadStatisticsInstanceType;
- private static LoadStatistics LoadStatisticsInstance;
-
- static {
- CarbonTimeStatisticsFactory.updateTimeStatisticsUtilStatus();
- LoadStatisticsInstance = genLoadStatisticsInstance();
- }
-
- private static void updateTimeStatisticsUtilStatus() {
- LoadStatisticsInstanceType = CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.ENABLE_DATA_LOADING_STATISTICS,
- CarbonCommonConstants.ENABLE_DATA_LOADING_STATISTICS_DEFAULT);
- }
-
- private static LoadStatistics genLoadStatisticsInstance() {
- switch (LoadStatisticsInstanceType.toLowerCase()) {
- case "false":
- return CarbonLoadStatisticsDummy.getInstance();
- case "true":
- return CarbonLoadStatisticsImpl.getInstance();
- default:
- return CarbonLoadStatisticsDummy.getInstance();
- }
- }
-
- public static LoadStatistics getLoadStatisticsInstance() {
- return LoadStatisticsInstance;
- }
-
-}