You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ch...@apache.org on 2016/06/30 17:42:00 UTC

[13/50] [abbrv] incubator-carbondata git commit: [BUG] Memory leak issue , Clean Up handling, Preserve handling during compaction (#748)

[BUG] Memory leak issue ,Clean Up handling,Preserve handling during compaction (#748)

* handling the cleanup of stale compacted segments in the compaction also.
* Fix problem in preserving segments during compaction
* Memory leak problem in query flow.

Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/24c47c20
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/24c47c20
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/24c47c20

Branch: refs/heads/master
Commit: 24c47c204426cdc51d6d3d13a47f4d351b687484
Parents: 322a77b
Author: ravikiran23 <ra...@gmail.com>
Authored: Sat Jun 25 18:56:14 2016 +0530
Committer: Venkata Ramana G <g....@gmail.com>
Committed: Sat Jun 25 18:56:14 2016 +0530

----------------------------------------------------------------------
 .../impl/InternalDetailQueryExecutor.java       |  6 +++
 .../result/iterator/RawResultIterator.java      |  2 +
 .../carbondata/spark/load/CarbonLoaderUtil.java | 26 ++++++++----
 .../spark/merger/CarbonDataMergerUtil.java      | 37 ++++-------------
 .../spark/rdd/CarbonDataRDDFactory.scala        | 43 ++++++++++++++++++--
 5 files changed, 74 insertions(+), 40 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/24c47c20/core/src/main/java/org/carbondata/query/carbon/executor/internal/impl/InternalDetailQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/carbon/executor/internal/impl/InternalDetailQueryExecutor.java b/core/src/main/java/org/carbondata/query/carbon/executor/internal/impl/InternalDetailQueryExecutor.java
index c7bfa6b..72a7f62 100644
--- a/core/src/main/java/org/carbondata/query/carbon/executor/internal/impl/InternalDetailQueryExecutor.java
+++ b/core/src/main/java/org/carbondata/query/carbon/executor/internal/impl/InternalDetailQueryExecutor.java
@@ -123,6 +123,12 @@ public class InternalDetailQueryExecutor implements InternalQueryExecutor {
       throw new QueryExecutionException(e);
     } finally {
       execService = null;
+      for (int currentSliceIndex : sliceIndexes) {
+        if (currentSliceIndex == -1) {
+          continue;
+        }
+        executionInfos.get(currentSliceIndex).setScannedResultProcessor(null);
+      }
     }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/24c47c20/core/src/main/java/org/carbondata/query/carbon/result/iterator/RawResultIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/query/carbon/result/iterator/RawResultIterator.java b/core/src/main/java/org/carbondata/query/carbon/result/iterator/RawResultIterator.java
index 27114f2..8977799 100644
--- a/core/src/main/java/org/carbondata/query/carbon/result/iterator/RawResultIterator.java
+++ b/core/src/main/java/org/carbondata/query/carbon/result/iterator/RawResultIterator.java
@@ -70,6 +70,7 @@ public class RawResultIterator extends CarbonIterator<Object[]> {
 
     if (null == batch || checkIfBatchIsProcessedCompletely(batch)) {
       if (detailRawQueryResultIterator.hasNext()) {
+        batch = null;
         batch = detailRawQueryResultIterator.next();
         counter = 0; // batch changed so reset the counter.
       } else {
@@ -102,6 +103,7 @@ public class RawResultIterator extends CarbonIterator<Object[]> {
         return null;
       }
     } else { // completed one batch.
+      batch = null;
       batch = detailRawQueryResultIterator.next();
       counter = 0;
     }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/24c47c20/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java b/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
index 427c7e5..0fb2621 100644
--- a/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
+++ b/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
@@ -285,18 +285,19 @@ public final class CarbonLoaderUtil {
     }
   }
 
-  public static void deletePartialLoadDataIfExist(CarbonLoadModel loadModel) throws IOException {
+  public static void deletePartialLoadDataIfExist(CarbonLoadModel loadModel,
+      final boolean isCompactionFlow) throws IOException {
     CarbonTable carbonTable = loadModel.getCarbonDataLoadSchema().getCarbonTable();
     String metaDataLocation = carbonTable.getMetaDataFilepath();
     SegmentStatusManager segmentStatusManager =
         new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier());
     LoadMetadataDetails[] details = segmentStatusManager.readLoadMetadata(metaDataLocation);
-    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(loadModel.getStorePath(),
-        carbonTable.getCarbonTableIdentifier());
+    CarbonTablePath carbonTablePath = CarbonStorePath
+        .getCarbonTablePath(loadModel.getStorePath(), carbonTable.getCarbonTableIdentifier());
     final List<String> loadFolders = new ArrayList<String>();
     for (LoadMetadataDetails loadMetadata : details) {
-      loadFolders.add(carbonTablePath.getCarbonDataDirectoryPath(loadMetadata.getPartitionCount(),
-          loadMetadata.getLoadName())
+      loadFolders.add(carbonTablePath
+          .getCarbonDataDirectoryPath(loadMetadata.getPartitionCount(), loadMetadata.getLoadName())
           .replace("\\", "/"));
     }
 
@@ -307,13 +308,22 @@ public final class CarbonLoaderUtil {
       if (FileFactory.isFileExist(partitionPath, fileType)) {
         CarbonFile carbonFile = FileFactory.getCarbonFile(partitionPath, fileType);
         CarbonFile[] listFiles = carbonFile.listFiles(new CarbonFileFilter() {
-          @Override
-          public boolean accept(CarbonFile path) {
+          @Override public boolean accept(CarbonFile path) {
             return !loadFolders.contains(path.getAbsolutePath().replace("\\", "/"));
           }
         });
         for (int k = 0; k < listFiles.length; k++) {
-          deleteStorePath(listFiles[k].getAbsolutePath());
+          String segmentId =
+              CarbonTablePath.DataPathUtil.getSegmentId(listFiles[k].getAbsolutePath() + "/dummy");
+          if (isCompactionFlow) {
+            if (segmentId.contains(".")) {
+              deleteStorePath(listFiles[k].getAbsolutePath());
+            }
+          } else {
+            if (!segmentId.contains(".")) {
+              deleteStorePath(listFiles[k].getAbsolutePath());
+            }
+          }
         }
       }
     }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/24c47c20/integration/spark/src/main/java/org/carbondata/spark/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/carbondata/spark/merger/CarbonDataMergerUtil.java b/integration/spark/src/main/java/org/carbondata/spark/merger/CarbonDataMergerUtil.java
index 7accb91..52e8a08 100644
--- a/integration/spark/src/main/java/org/carbondata/spark/merger/CarbonDataMergerUtil.java
+++ b/integration/spark/src/main/java/org/carbondata/spark/merger/CarbonDataMergerUtil.java
@@ -588,39 +588,20 @@ public final class CarbonDataMergerUtil {
       }
     }
 
-    // handle the retaining of valid loads,
-
     // check if valid list is big enough for removing the number of seg to be retained.
-    if (validList.size() > numberOfSegToBeRetained) {
-
-      // after the sort remove the loads from the last as per the retaining count.
-      Collections.sort(validList, new Comparator<LoadMetadataDetails>() {
-
-        @Override public int compare(LoadMetadataDetails seg1, LoadMetadataDetails seg2) {
-          double segNumber1 = Double.parseDouble(seg1.getLoadName());
-          double segNumber2 = Double.parseDouble(seg2.getLoadName());
-
-          if ((segNumber1 - segNumber2) < 0) {
-            return -1;
-          } else if ((segNumber1 - segNumber2) > 0) {
-            return 1;
-          }
-          return 0;
-
-        }
-      });
-
-      for (int i = 0; i < numberOfSegToBeRetained; i++) {
-
-        // remove last segment
-        validList.remove(validList.size() - 1);
+    // last element
+    int removingIndex = validList.size() - 1;
 
+    for (int i = validList.size(); i > 0; i--) {
+      if (numberOfSegToBeRetained == 0) {
+        break;
       }
-      return validList;
+      // remove last segment
+      validList.remove(removingIndex--);
+      numberOfSegToBeRetained--;
     }
+    return validList;
 
-    // case where there is no 2 loads available for merging.
-    return new ArrayList<LoadMetadataDetails>(0);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/24c47c20/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index e534e3d..6f44aef 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -369,6 +369,18 @@ object CarbonDataRDDFactory extends Logging {
     readLoadMetadataDetails(carbonLoadModel, hdfsStoreLocation)
     var segList: util.List[LoadMetadataDetails] = carbonLoadModel.getLoadMetadataDetails
 
+    // clean up of the stale segments.
+    try {
+      CarbonLoaderUtil.deletePartialLoadDataIfExist(carbonLoadModel, true)
+    }
+    catch {
+      case e: Exception =>
+        logger
+          .error("Exception in compaction thread while clean up of stale segments " + e
+            .getMessage
+          )
+    }
+
     var loadsToMerge = CarbonDataMergerUtil.identifySegmentsToBeMerged(
       hdfsStoreLocation,
       carbonLoadModel,
@@ -384,11 +396,25 @@ object CarbonDataRDDFactory extends Logging {
         override def run(): Unit = {
 
           while (loadsToMerge.size() > 1) {
-
+          // Deleting the any partially loaded data if present.
+          // in some case the segment folder which is present in store will not have entry in
+          // status.
+          // so deleting those folders.
+          try {
+            CarbonLoaderUtil.deletePartialLoadDataIfExist(carbonLoadModel, true)
+          }
+          catch {
+            case e: Exception =>
+              logger
+                .error("Exception in compaction thread while clean up of stale segments " + e
+                  .getMessage
+                )
+          }
             val futureList: util.List[Future[Void]] = new util.ArrayList[Future[Void]](
               CarbonCommonConstants
                 .DEFAULT_COLLECTION_SIZE
             )
+
             scanSegmentsAndSubmitJob(futureList)
 
             futureList.asScala.foreach(future => {
@@ -425,7 +451,8 @@ object CarbonDataRDDFactory extends Logging {
 
     /**
      * This will scan all the segments and submit the loads to be merged into the executor.
-     * @param futureList
+      *
+      * @param futureList
      */
     def scanSegmentsAndSubmitJob(futureList: util.List[Future[Void]]): Unit = {
       breakable {
@@ -579,8 +606,16 @@ object CarbonDataRDDFactory extends Logging {
       // Deleting the any partially loaded data if present.
       // in some case the segment folder which is present in store will not have entry in status.
       // so deleting those folders.
-      CarbonLoaderUtil.deletePartialLoadDataIfExist(carbonLoadModel)
-
+      try {
+        CarbonLoaderUtil.deletePartialLoadDataIfExist(carbonLoadModel, false)
+      }
+      catch {
+        case e: Exception =>
+          logger
+            .error("Exception in data load while clean up of stale segments " + e
+              .getMessage
+            )
+      }
 
       // reading the start time of data load.
       val loadStartTime = CarbonLoaderUtil.readCurrentTime()