You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/07/09 07:18:26 UTC
carbondata git commit: [CARBONDATA-2658][DataLoad] Fix bugs in
spilling in-memory pages
Repository: carbondata
Updated Branches:
refs/heads/master c2c5b18ba -> c3bc1ba10
[CARBONDATA-2658][DataLoad] Fix bugs in spilling in-memory pages
the parameter carbon.load.sortMemory.spill.percentage configured the value range 0-100,according to configuration merge and spill in-memory pages to disk
This closes #2414
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c3bc1ba1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c3bc1ba1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c3bc1ba1
Branch: refs/heads/master
Commit: c3bc1ba10ef4c468f199fdea6e38d8b197908fcd
Parents: c2c5b18
Author: ndwangsen <lu...@huawei.com>
Authored: Fri Jun 22 12:02:36 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Mon Jul 9 15:18:12 2018 +0800
----------------------------------------------------------------------
.../processing/loading/sort/unsafe/UnsafeSortDataRows.java | 1 -
.../sort/unsafe/merger/UnsafeIntermediateMerger.java | 9 +++++++++
2 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3bc1ba1/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java
index fb0bcc3..76532f0 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java
@@ -149,7 +149,6 @@ public class UnsafeSortDataRows {
if (isMemoryAvailable) {
UnsafeSortMemoryManager.INSTANCE.allocateDummyMemory(baseBlock.size());
} else {
- LOGGER.info("trigger in-memory merge and spill for table " + parameters.getTableName());
// merge and spill in-memory pages to disk if memory is not enough
unsafeInMemoryIntermediateFileMerger.tryTriggerInmemoryMerging(true);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3bc1ba1/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java
index bd2f5d9..e6e24ec 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java
@@ -84,6 +84,14 @@ public class UnsafeIntermediateMerger {
Integer spillPercentage = CarbonProperties.getInstance().getSortMemorySpillPercentage();
this.spillSizeInSortMemory =
UnsafeSortMemoryManager.INSTANCE.getUsableMemory() * spillPercentage / 100;
+ // get memory chunk size
+ long inMemoryChunkSizeInMB = CarbonProperties.getInstance().getSortMemoryChunkSizeInMB();
+ if (spillSizeInSortMemory < inMemoryChunkSizeInMB * 1024 * 1024) {
+ LOGGER.warn("the configure spill size is " + spillSizeInSortMemory +
+ " less than the page size " + inMemoryChunkSizeInMB * 1024 * 1024 +
+ ",so no merge and spill in-memory pages to disk");
+ }
+
}
public void addDataChunkToMerge(UnsafeCarbonRowPage rowPage) {
@@ -147,6 +155,7 @@ public class UnsafeIntermediateMerger {
UnsafeCarbonRowPage page = iter.next();
if (!spillDisk || sizeAdded + page.getDataBlock().size() < this.spillSizeInSortMemory) {
pages2Merge.add(page);
+ sizeAdded += page.getDataBlock().size();
totalRows2Merge += page.getBuffer().getActualSize();
iter.remove();
} else {