You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2016/06/30 17:42:01 UTC
[14/50] [abbrv] incubator-carbondata git commit: [Bug] Added comments
for carbon.properties.template and Carbonindex file cleanup fix (#749)
[Bug] Added comments for carbon.properties.template and Carbonindex file cleanup fix (#749)
* Added comments for carbon.properties.template
* clean files not cleaning .carbonindex files
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/720e8d61
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/720e8d61
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/720e8d61
Branch: refs/heads/master
Commit: 720e8d6152e91be7276be5a7c1317d17caf5b30b
Parents: 24c47c2
Author: nareshpr <pr...@gmail.com>
Authored: Sat Jun 25 19:53:02 2016 +0530
Committer: Venkata Ramana G <g....@gmail.com>
Committed: Sat Jun 25 19:53:02 2016 +0530
----------------------------------------------------------------------
conf/carbon.properties.template | 42 ++++++++++----------
.../core/carbon/path/CarbonTablePath.java | 14 +++++++
.../core/constants/CarbonCommonConstants.java | 12 +++---
.../spark/load/DeleteLoadFolders.java | 5 ++-
.../store/SingleThreadFinalSortFilesMerger.java | 6 +--
5 files changed, 47 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/720e8d61/conf/carbon.properties.template
----------------------------------------------------------------------
diff --git a/conf/carbon.properties.template b/conf/carbon.properties.template
index 314320c..8dd6598 100644
--- a/conf/carbon.properties.template
+++ b/conf/carbon.properties.template
@@ -10,13 +10,13 @@ carbon.kettle.home=$<SPARK_HOME>/carbonlib/carbonplugins
#################### Performance Configuration ##################
######## DataLoading Configuration ########
-#File read buffer size used during sorting:MIN=:MAX=
+#File read buffer size used during sorting(in MB) :MIN=1:MAX=100
carbon.sort.file.buffer.size=20
-#Rowset size exchanged between data load graph steps.:MIN=:MAX=
+#Rowset size exchanged between data load graph steps :MIN=500:MAX=1000000
carbon.graph.rowset.size=100000
-#Number of cores to be used while data loading:MIN=:MAX=
+#Number of cores to be used while data loading
carbon.number.of.cores.while.loading=6
-#CARBON sort size.:MIN=:MAX=
+#Record count to sort and write to temp intermediate files
carbon.sort.size=500000
#Algorithm for hashmap for hashkey calculation
carbon.enableXXHash=true
@@ -27,43 +27,45 @@ carbon.enableXXHash=true
#enable prefetch of data during merge sort while reading data from sort temp files in data loading
#carbon.merge.sort.prefetch=true
######## Compaction Configuration ########
-#Number of cores to be used while compacting:MIN=:MAX=
+#Number of cores to be used while compacting
carbon.number.of.cores.while.compacting=2
-#default minor compaction in MBs
-carbon.minor.compaction.size=256
-#default major compaction in MBs
+#For minor compaction, Number of segments to be merged in stage 1, number of compacted segments to be merged in stage 2.
+carbon.compaction.level.threshold=4,3
+#default size (in MB) for major compaction to be triggered
carbon.major.compaction.size=1024
######## Query Configuration ########
-#Number of cores to be used.:MIN=:MAX=
+#Number of cores to be used while querying
carbon.number.of.cores=4
-#Carbon Inmemory record size:MIN=:MAX=
-carbon.inmemory.record.size=100000
+#Number of records to be in memory while querying :MIN=100000:MAX=240000
+carbon.inmemory.record.size=120000
#Improves the performance of filter query
carbon.enable.quick.filter=false
+##number of core to load the blocks in driver
+#no.of.cores.to.load.blocks.in.driver=10
#################### Extra Configuration ##################
##Timestamp format of input data used for timestamp data type.
#carbon.timestamp.format=yyyy-MM-dd HH:mm:ss
######## Dataload Configuration ########
-######File write buffer size used during sorting.
+##File write buffer size used during sorting.
#carbon.sort.file.write.buffer.size=10485760
+##Locking mechanism for data loading on a table
+#carbon.lock.type=LOCALLOCK
##Minimum no of intermediate files after which sort merged to be started.
#carbon.sort.intermediate.files.limit=20
##space reserved in percentage for writing block meta data in carbon data file
#carbon.block.meta.size.reserved.percentage=10
##csv reading buffer size.
#carbon.csv.read.buffersize.byte=1048576
-##space reserved in percentage for writing block meta data in carbon data file
-#carbon.block.meta.size.reserved.percentage=10
-##High Cardinality value
+##To identify and apply compression for non-high cardinality columns
#high.cardinality.value=100000
-##CARBON maximum no of threads used for sorting.
-#carbon.max.thread.for.sorting=3
+##maximum no of threads used for reading intermediate files for final merging.
+#carbon.merge.sort.reader.thread=3
##Carbon blocklet size. Note: this configuration cannot be change once store is generated
#carbon.blocklet.size=120000
-##How to times retry to get the lock
+##number of retries to get the metadata lock for loading data to table
#carbon.load.metadata.lock.retries=3
-##Maximum number of blocklets written in a single file.:Min=1:Max=1000
+##Maximum number of blocklets written in a single file :Min=1:Max=1000
#carbon.max.file.size=100
##Interval between the retries to get the lock
#carbon.load.metadata.lock.retry.timeout.sec=5
@@ -83,8 +85,6 @@ carbon.enable.quick.filter=false
#max.query.execution.time=60
##Min max is feature added to enhance query performance. To disable this feature, make it false.
#carbon.enableMinMax=true
-##number of core to load the blocks in driver
-#no.of.cores.to.load.blocks.in.driver=10
######## Global Dictionary Configurations ########
##To enable/disable identify high cardinality during first data loading
#high.cardinality.identify.enable=true
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/720e8d61/core/src/main/java/org/carbondata/core/carbon/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/carbon/path/CarbonTablePath.java b/core/src/main/java/org/carbondata/core/carbon/path/CarbonTablePath.java
index 8dcd207..5f862bc 100644
--- a/core/src/main/java/org/carbondata/core/carbon/path/CarbonTablePath.java
+++ b/core/src/main/java/org/carbondata/core/carbon/path/CarbonTablePath.java
@@ -98,6 +98,20 @@ public class CarbonTablePath extends Path {
}
/**
+ * check if it is carbon index file matching extension
+ *
+ * @param fileNameWithPath
+ * @return boolean
+ */
+ public static boolean isCarbonIndexFile(String fileNameWithPath) {
+ int pos = fileNameWithPath.lastIndexOf('.');
+ if (pos != -1) {
+ return fileNameWithPath.substring(pos).startsWith(INDEX_FILE_EXT);
+ }
+ return false;
+ }
+
+ /**
* gets table path
*/
public String getPath() {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/720e8d61/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
index a120d00..473e764 100644
--- a/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/carbondata/core/constants/CarbonCommonConstants.java
@@ -250,7 +250,7 @@ public final class CarbonCommonConstants {
/**
* SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE
*/
- public static final String SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE = "10";
+ public static final String SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE = "20";
/**
* MERGERD_EXTENSION
*/
@@ -349,13 +349,13 @@ public final class CarbonCommonConstants {
*/
public static final String SORT_TEMP_FILE_EXT = ".sorttemp";
/**
- * CARBON_MAX_THREAD_FOR_SORTING
+ * CARBON_MERGE_SORT_READER_THREAD
*/
- public static final String CARBON_MAX_THREAD_FOR_SORTING = "carbon.max.thread.for.sorting";
+ public static final String CARBON_MERGE_SORT_READER_THREAD = "carbon.merge.sort.reader.thread";
/**
- * CARBON_MAX_THREAD_FOR_SORTING
+ * CARBON_MERGE_SORT_READER_THREAD_DEFAULTVALUE
*/
- public static final String CARBON_MAX_THREAD_FOR_SORTING_DEFAULTVALUE = "2";
+ public static final String CARBON_MERGE_SORT_READER_THREAD_DEFAULTVALUE = "3";
/**
* IS_SORT_TEMP_FILE_COMPRESSION_ENABLED
*/
@@ -645,7 +645,7 @@ public final class CarbonCommonConstants {
/**
* xxhash algorithm property for hashmap. Default value false
*/
- public static final String ENABLE_XXHASH_DEFAULT = "false";
+ public static final String ENABLE_XXHASH_DEFAULT = "true";
/**
* default charset to be used for reading and writing
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/720e8d61/integration/spark/src/main/java/org/carbondata/spark/load/DeleteLoadFolders.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/carbondata/spark/load/DeleteLoadFolders.java b/integration/spark/src/main/java/org/carbondata/spark/load/DeleteLoadFolders.java
index 1bb48d4..01d785a 100644
--- a/integration/spark/src/main/java/org/carbondata/spark/load/DeleteLoadFolders.java
+++ b/integration/spark/src/main/java/org/carbondata/spark/load/DeleteLoadFolders.java
@@ -38,6 +38,7 @@ import java.util.List;
import org.carbondata.common.logging.LogService;
import org.carbondata.common.logging.LogServiceFactory;
import org.carbondata.core.carbon.path.CarbonStorePath;
+import org.carbondata.core.carbon.path.CarbonTablePath;
import org.carbondata.core.constants.CarbonCommonConstants;
import org.carbondata.core.datastorage.store.filesystem.CarbonFile;
import org.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
@@ -180,8 +181,8 @@ public final class DeleteLoadFolders {
CarbonFile[] filesToBeDeleted = file.listFiles(new CarbonFileFilter() {
@Override public boolean accept(CarbonFile file) {
- return (file.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT) || file.getName()
- .endsWith(CarbonCommonConstants.MEASUREMETADATA_FILE_EXT));
+ return (CarbonTablePath.isCarbonDataFile(file.getName())
+ || CarbonTablePath.isCarbonIndexFile(file.getName()));
}
});
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/720e8d61/processing/src/main/java/org/carbondata/processing/store/SingleThreadFinalSortFilesMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/carbondata/processing/store/SingleThreadFinalSortFilesMerger.java b/processing/src/main/java/org/carbondata/processing/store/SingleThreadFinalSortFilesMerger.java
index 9096b07..85f7c54 100644
--- a/processing/src/main/java/org/carbondata/processing/store/SingleThreadFinalSortFilesMerger.java
+++ b/processing/src/main/java/org/carbondata/processing/store/SingleThreadFinalSortFilesMerger.java
@@ -162,11 +162,11 @@ public class SingleThreadFinalSortFilesMerger {
int maxThreadForSorting = 0;
try {
maxThreadForSorting = Integer.parseInt(CarbonProperties.getInstance()
- .getProperty(CarbonCommonConstants.CARBON_MAX_THREAD_FOR_SORTING,
- CarbonCommonConstants.CARBON_MAX_THREAD_FOR_SORTING_DEFAULTVALUE));
+ .getProperty(CarbonCommonConstants.CARBON_MERGE_SORT_READER_THREAD,
+ CarbonCommonConstants.CARBON_MERGE_SORT_READER_THREAD_DEFAULTVALUE));
} catch (NumberFormatException e) {
maxThreadForSorting =
- Integer.parseInt(CarbonCommonConstants.CARBON_MAX_THREAD_FOR_SORTING_DEFAULTVALUE);
+ Integer.parseInt(CarbonCommonConstants.CARBON_MERGE_SORT_READER_THREAD_DEFAULTVALUE);
}
ExecutorService service = Executors.newFixedThreadPool(maxThreadForSorting);