You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/08/09 18:26:23 UTC
[39/47] carbondata git commit: [CARBONDATA-2807] Fixed data load
performance issue in Intermediate merger When number of records are high
[CARBONDATA-2807] Fixed data load performance issue in Intermediate merger When number of records are high
Problem: Data Loading is taking more time when number of records are high.
Root cause: As number of records are high intermediate merger is taking more time.
Solution: Checking the number of files present in file list is done is synchronized block because of this
each intermediate request is taking sometime and when number of records are high it impacting overall data loading performance
This closes #2588
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/dc106c49
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/dc106c49
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/dc106c49
Branch: refs/heads/branch-1.4
Commit: dc106c49a7bb1b10c3a1bf4ad1cde28570814db2
Parents: 7a65ad1
Author: kumarvishal09 <ku...@gmail.com>
Authored: Mon Jul 30 21:21:09 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Aug 9 23:51:17 2018 +0530
----------------------------------------------------------------------
.../unsafe/merger/UnsafeIntermediateMerger.java | 20 +++++++-------------
1 file changed, 7 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/dc106c49/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java
index 0b4eae2..18f35d3 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java
@@ -37,8 +37,6 @@ import org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage;
import org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException;
import org.apache.carbondata.processing.sort.sortdata.SortParameters;
-import org.apache.commons.collections.list.SynchronizedList;
-
/**
* It does mergesort intermediate files to big file.
*/
@@ -77,8 +75,7 @@ public class UnsafeIntermediateMerger {
this.mergedPages = new ArrayList<>();
this.executorService = Executors.newFixedThreadPool(parameters.getNumberOfCores(),
new CarbonThreadFactory("UnsafeIntermediatePool:" + parameters.getTableName()));
- this.procFiles =
- SynchronizedList.decorate(new ArrayList<File>(CarbonCommonConstants.CONSTANT_SIZE_TEN));
+ this.procFiles = new ArrayList<>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
this.mergerTask = new ArrayList<>();
Integer spillPercentage = CarbonProperties.getInstance().getSortMemorySpillPercentage();
@@ -111,18 +108,15 @@ public class UnsafeIntermediateMerger {
}
public void startFileMergingIfPossible() {
- File[] fileList = null;
- synchronized (lockObject) {
- if (procFiles.size() >= parameters.getNumberOfIntermediateFileToBeMerged()) {
+ File[] fileList;
+ if (procFiles.size() >= parameters.getNumberOfIntermediateFileToBeMerged()) {
+ synchronized (lockObject) {
fileList = procFiles.toArray(new File[procFiles.size()]);
this.procFiles = new ArrayList<File>();
- if (LOGGER.isDebugEnabled()) {
- LOGGER
- .debug("Submitting request for intermediate merging no of files: " + fileList.length);
- }
}
- }
- if (null != fileList) {
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Sumitting request for intermediate merging no of files: " + fileList.length);
+ }
startIntermediateMerging(fileList);
}
}