You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/06/22 01:28:30 UTC
[1/2] carbondata git commit: Problem: Memory leak in case of
compaction when unsafe is true
Repository: carbondata
Updated Branches:
refs/heads/master 3a6f8ca2c -> 56e56c037
Problem: Memory leak in case of compaction when unsafe is true
Analysis: In case of compaction, queryExecutor object is formed for multiple blocks but the objects are not retained and finish method is called only on the last instance created for query executor. Due to this the memory allocated to precious objects is not released which can lead to out of memory issue.
Fix: Add all the object to a list and call finish method on all the objects so that memory gets released
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c0facf28
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c0facf28
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c0facf28
Branch: refs/heads/master
Commit: c0facf287ebf88277f0fd96212c76d96984dea73
Parents: 3a6f8ca
Author: manishgupta88 <to...@gmail.com>
Authored: Wed Jun 21 20:09:12 2017 +0530
Committer: jackylk <ja...@huawei.com>
Committed: Thu Jun 22 09:27:50 2017 +0800
----------------------------------------------------------------------
.../processing/merger/CarbonCompactionExecutor.java | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/c0facf28/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
index c00fe2e..94bfbe4 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonCompactionExecutor.java
@@ -58,7 +58,7 @@ public class CarbonCompactionExecutor {
private final Map<String, List<DataFileFooter>> dataFileMetadataSegMapping;
private final SegmentProperties destinationSegProperties;
private final Map<String, TaskBlockInfo> segmentMapping;
- private QueryExecutor queryExecutor;
+ private List<QueryExecutor> queryExecutorList;
private CarbonTable carbonTable;
private QueryModel queryModel;
@@ -86,6 +86,7 @@ public class CarbonCompactionExecutor {
this.carbonTable = carbonTable;
this.dataFileMetadataSegMapping = dataFileMetadataSegMapping;
this.restructuredBlockExists = restructuredBlockExists;
+ queryExecutorList = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
}
/**
@@ -156,7 +157,8 @@ public class CarbonCompactionExecutor {
private CarbonIterator<BatchResult> executeBlockList(List<TableBlockInfo> blockList)
throws QueryExecutionException, IOException {
queryModel.setTableBlockInfos(blockList);
- this.queryExecutor = QueryExecutorFactory.getQueryExecutor(queryModel);
+ QueryExecutor queryExecutor = QueryExecutorFactory.getQueryExecutor(queryModel);
+ queryExecutorList.add(queryExecutor);
return queryExecutor.execute(queryModel);
}
@@ -166,7 +168,9 @@ public class CarbonCompactionExecutor {
*/
public void finish() {
try {
- queryExecutor.finish();
+ for (QueryExecutor queryExecutor : queryExecutorList) {
+ queryExecutor.finish();
+ }
} catch (QueryExecutionException e) {
LOGGER.error(e, "Problem while finish: ");
}
[2/2] carbondata git commit: [CARBONDATA-1212] Memory leak in case of
compaction when unsafe is true This closes #1074
Posted by ja...@apache.org.
[CARBONDATA-1212] Memory leak in case of compaction when unsafe is true This closes #1074
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/56e56c03
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/56e56c03
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/56e56c03
Branch: refs/heads/master
Commit: 56e56c037d3bb23fbf73686eda10bf3a9d9b5396
Parents: 3a6f8ca c0facf2
Author: jackylk <ja...@huawei.com>
Authored: Thu Jun 22 09:28:17 2017 +0800
Committer: jackylk <ja...@huawei.com>
Committed: Thu Jun 22 09:28:17 2017 +0800
----------------------------------------------------------------------
.../processing/merger/CarbonCompactionExecutor.java | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------