You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2016/06/30 17:42:00 UTC
[13/50] [abbrv] incubator-carbondata git commit: [BUG] Memory leak
issue , Clean Up handling, Preserve handling during compaction (#748)
[BUG] Memory leak issue ,Clean Up handling,Preserve handling during compaction (#748)
* handling the cleanup of stale compacted segments in the compaction also.
* Fix problem in preserving segments during compaction
* Memory leak problem in query flow.
Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/24c47c20
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/24c47c20
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/24c47c20
Branch: refs/heads/master
Commit: 24c47c204426cdc51d6d3d13a47f4d351b687484
Parents: 322a77b
Author: ravikiran23 <ra...@gmail.com>
Authored: Sat Jun 25 18:56:14 2016 +0530
Committer: Venkata Ramana G <g....@gmail.com>
Committed: Sat Jun 25 18:56:14 2016 +0530
----------------------------------------------------------------------
.../impl/InternalDetailQueryExecutor.java | 6 +++
.../result/iterator/RawResultIterator.java | 2 +
.../carbondata/spark/load/CarbonLoaderUtil.java | 26 ++++++++----
.../spark/merger/CarbonDataMergerUtil.java | 37 ++++-------------
.../spark/rdd/CarbonDataRDDFactory.scala | 43 ++++++++++++++++++--
5 files changed, 74 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/24c47c20/core/src/main/java/org/carbondata/query/carbon/executor/internal/impl/InternalDetailQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/carbon/executor/internal/impl/InternalDetailQueryExecutor.java b/core/src/main/java/org/carbondata/query/carbon/executor/internal/impl/InternalDetailQueryExecutor.java
index c7bfa6b..72a7f62 100644
--- a/core/src/main/java/org/carbondata/query/carbon/executor/internal/impl/InternalDetailQueryExecutor.java
+++ b/core/src/main/java/org/carbondata/query/carbon/executor/internal/impl/InternalDetailQueryExecutor.java
@@ -123,6 +123,12 @@ public class InternalDetailQueryExecutor implements InternalQueryExecutor {
throw new QueryExecutionException(e);
} finally {
execService = null;
+ for (int currentSliceIndex : sliceIndexes) {
+ if (currentSliceIndex == -1) {
+ continue;
+ }
+ executionInfos.get(currentSliceIndex).setScannedResultProcessor(null);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/24c47c20/core/src/main/java/org/carbondata/query/carbon/result/iterator/RawResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/carbon/result/iterator/RawResultIterator.java b/core/src/main/java/org/carbondata/query/carbon/result/iterator/RawResultIterator.java
index 27114f2..8977799 100644
--- a/core/src/main/java/org/carbondata/query/carbon/result/iterator/RawResultIterator.java
+++ b/core/src/main/java/org/carbondata/query/carbon/result/iterator/RawResultIterator.java
@@ -70,6 +70,7 @@ public class RawResultIterator extends CarbonIterator<Object[]> {
if (null == batch || checkIfBatchIsProcessedCompletely(batch)) {
if (detailRawQueryResultIterator.hasNext()) {
+ batch = null;
batch = detailRawQueryResultIterator.next();
counter = 0; // batch changed so reset the counter.
} else {
@@ -102,6 +103,7 @@ public class RawResultIterator extends CarbonIterator<Object[]> {
return null;
}
} else { // completed one batch.
+ batch = null;
batch = detailRawQueryResultIterator.next();
counter = 0;
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/24c47c20/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java b/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
index 427c7e5..0fb2621 100644
--- a/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
+++ b/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
@@ -285,18 +285,19 @@ public final class CarbonLoaderUtil {
}
}
- public static void deletePartialLoadDataIfExist(CarbonLoadModel loadModel) throws IOException {
+ public static void deletePartialLoadDataIfExist(CarbonLoadModel loadModel,
+ final boolean isCompactionFlow) throws IOException {
CarbonTable carbonTable = loadModel.getCarbonDataLoadSchema().getCarbonTable();
String metaDataLocation = carbonTable.getMetaDataFilepath();
SegmentStatusManager segmentStatusManager =
new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier());
LoadMetadataDetails[] details = segmentStatusManager.readLoadMetadata(metaDataLocation);
- CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(loadModel.getStorePath(),
- carbonTable.getCarbonTableIdentifier());
+ CarbonTablePath carbonTablePath = CarbonStorePath
+ .getCarbonTablePath(loadModel.getStorePath(), carbonTable.getCarbonTableIdentifier());
final List<String> loadFolders = new ArrayList<String>();
for (LoadMetadataDetails loadMetadata : details) {
- loadFolders.add(carbonTablePath.getCarbonDataDirectoryPath(loadMetadata.getPartitionCount(),
- loadMetadata.getLoadName())
+ loadFolders.add(carbonTablePath
+ .getCarbonDataDirectoryPath(loadMetadata.getPartitionCount(), loadMetadata.getLoadName())
.replace("\\", "/"));
}
@@ -307,13 +308,22 @@ public final class CarbonLoaderUtil {
if (FileFactory.isFileExist(partitionPath, fileType)) {
CarbonFile carbonFile = FileFactory.getCarbonFile(partitionPath, fileType);
CarbonFile[] listFiles = carbonFile.listFiles(new CarbonFileFilter() {
- @Override
- public boolean accept(CarbonFile path) {
+ @Override public boolean accept(CarbonFile path) {
return !loadFolders.contains(path.getAbsolutePath().replace("\\", "/"));
}
});
for (int k = 0; k < listFiles.length; k++) {
- deleteStorePath(listFiles[k].getAbsolutePath());
+ String segmentId =
+ CarbonTablePath.DataPathUtil.getSegmentId(listFiles[k].getAbsolutePath() + "/dummy");
+ if (isCompactionFlow) {
+ if (segmentId.contains(".")) {
+ deleteStorePath(listFiles[k].getAbsolutePath());
+ }
+ } else {
+ if (!segmentId.contains(".")) {
+ deleteStorePath(listFiles[k].getAbsolutePath());
+ }
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/24c47c20/integration/spark/src/main/java/org/carbondata/spark/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/carbondata/spark/merger/CarbonDataMergerUtil.java b/integration/spark/src/main/java/org/carbondata/spark/merger/CarbonDataMergerUtil.java
index 7accb91..52e8a08 100644
--- a/integration/spark/src/main/java/org/carbondata/spark/merger/CarbonDataMergerUtil.java
+++ b/integration/spark/src/main/java/org/carbondata/spark/merger/CarbonDataMergerUtil.java
@@ -588,39 +588,20 @@ public final class CarbonDataMergerUtil {
}
}
- // handle the retaining of valid loads,
-
// check if valid list is big enough for removing the number of seg to be retained.
- if (validList.size() > numberOfSegToBeRetained) {
-
- // after the sort remove the loads from the last as per the retaining count.
- Collections.sort(validList, new Comparator<LoadMetadataDetails>() {
-
- @Override public int compare(LoadMetadataDetails seg1, LoadMetadataDetails seg2) {
- double segNumber1 = Double.parseDouble(seg1.getLoadName());
- double segNumber2 = Double.parseDouble(seg2.getLoadName());
-
- if ((segNumber1 - segNumber2) < 0) {
- return -1;
- } else if ((segNumber1 - segNumber2) > 0) {
- return 1;
- }
- return 0;
-
- }
- });
-
- for (int i = 0; i < numberOfSegToBeRetained; i++) {
-
- // remove last segment
- validList.remove(validList.size() - 1);
+ // last element
+ int removingIndex = validList.size() - 1;
+ for (int i = validList.size(); i > 0; i--) {
+ if (numberOfSegToBeRetained == 0) {
+ break;
}
- return validList;
+ // remove last segment
+ validList.remove(removingIndex--);
+ numberOfSegToBeRetained--;
}
+ return validList;
- // case where there is no 2 loads available for merging.
- return new ArrayList<LoadMetadataDetails>(0);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/24c47c20/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index e534e3d..6f44aef 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -369,6 +369,18 @@ object CarbonDataRDDFactory extends Logging {
readLoadMetadataDetails(carbonLoadModel, hdfsStoreLocation)
var segList: util.List[LoadMetadataDetails] = carbonLoadModel.getLoadMetadataDetails
+ // clean up of the stale segments.
+ try {
+ CarbonLoaderUtil.deletePartialLoadDataIfExist(carbonLoadModel, true)
+ }
+ catch {
+ case e: Exception =>
+ logger
+ .error("Exception in compaction thread while clean up of stale segments " + e
+ .getMessage
+ )
+ }
+
var loadsToMerge = CarbonDataMergerUtil.identifySegmentsToBeMerged(
hdfsStoreLocation,
carbonLoadModel,
@@ -384,11 +396,25 @@ object CarbonDataRDDFactory extends Logging {
override def run(): Unit = {
while (loadsToMerge.size() > 1) {
-
+ // Deleting the any partially loaded data if present.
+ // in some case the segment folder which is present in store will not have entry in
+ // status.
+ // so deleting those folders.
+ try {
+ CarbonLoaderUtil.deletePartialLoadDataIfExist(carbonLoadModel, true)
+ }
+ catch {
+ case e: Exception =>
+ logger
+ .error("Exception in compaction thread while clean up of stale segments " + e
+ .getMessage
+ )
+ }
val futureList: util.List[Future[Void]] = new util.ArrayList[Future[Void]](
CarbonCommonConstants
.DEFAULT_COLLECTION_SIZE
)
+
scanSegmentsAndSubmitJob(futureList)
futureList.asScala.foreach(future => {
@@ -425,7 +451,8 @@ object CarbonDataRDDFactory extends Logging {
/**
* This will scan all the segments and submit the loads to be merged into the executor.
- * @param futureList
+ *
+ * @param futureList
*/
def scanSegmentsAndSubmitJob(futureList: util.List[Future[Void]]): Unit = {
breakable {
@@ -579,8 +606,16 @@ object CarbonDataRDDFactory extends Logging {
// Deleting the any partially loaded data if present.
// in some case the segment folder which is present in store will not have entry in status.
// so deleting those folders.
- CarbonLoaderUtil.deletePartialLoadDataIfExist(carbonLoadModel)
-
+ try {
+ CarbonLoaderUtil.deletePartialLoadDataIfExist(carbonLoadModel, false)
+ }
+ catch {
+ case e: Exception =>
+ logger
+ .error("Exception in data load while clean up of stale segments " + e
+ .getMessage
+ )
+ }
// reading the start time of data load.
val loadStartTime = CarbonLoaderUtil.readCurrentTime()