[carbondata] branch master updated: [CARBONDATA-3360]fix NullPointerException in delete and clean files operation

[ra...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

ravipesala pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git


The following commit(s) were added to refs/heads/master by this push:
     new 3268a45  [CARBONDATA-3360]fix NullPointerException in delete and clean files operation
3268a45 is described below

commit 3268a456c31fa113b4a3c8aa40ba26fff5728c6e
Author: akashrn5 <ak...@gmail.com>
AuthorDate: Fri Apr 26 12:00:41 2019 +0530

    [CARBONDATA-3360]fix NullPointerException in delete and clean files operation
    
    Problem:
    when delete is failed due to hdfs quota exceeded or disk space is full, then tableUpdateStatus.write will be present in store.
    So after that if clean files operation is done, we were trying to assign null to primitive type long, which will throw runtime exception, and .write file will not be deleted, since we consider it as invalid file.
    
    Solution:
    if .write file is present, then we do not fail clean files, we check for max query timeout for tableUpdateStatus.write file and then delete these .write files for any clean files operation after that.
    
    This closes #3191
---
 .../carbondata/core/mutate/CarbonUpdateUtil.java   | 48 +++++++++++++++-------
 1 file changed, 34 insertions(+), 14 deletions(-)

diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
index a632f03..beaf1a0 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
@@ -673,7 +673,8 @@ public class CarbonUpdateUtil {
   private static boolean compareTimestampsAndDelete(
       CarbonFile invalidFile,
       boolean forceDelete, boolean isUpdateStatusFile) {
-    long fileTimestamp = 0L;
+    boolean isDeleted = false;
+    Long fileTimestamp;
 
     if (isUpdateStatusFile) {
       fileTimestamp = CarbonUpdateUtil.getTimeStampAsLong(invalidFile.getName()
@@ -683,21 +684,40 @@ public class CarbonUpdateUtil {
               CarbonTablePath.DataFileUtil.getTimeStampFromFileName(invalidFile.getName()));
     }
 
-    // if the timestamp of the file is more than the current time by query execution timeout.
-    // then delete that file.
-    if (CarbonUpdateUtil.isMaxQueryTimeoutExceeded(fileTimestamp) || forceDelete) {
-      // delete the files.
-      try {
-        LOGGER.info("deleting the invalid file : " + invalidFile.getName());
-        CarbonUtil.deleteFoldersAndFiles(invalidFile);
-        return true;
-      } catch (IOException e) {
-        LOGGER.error("error in clean up of compacted files." + e.getMessage(), e);
-      } catch (InterruptedException e) {
-        LOGGER.error("error in clean up of compacted files." + e.getMessage(), e);
+    // This check is because, when there are some invalid files like tableStatusUpdate.write files
+    // present in store [[which can happen during delete or update if the disk is full or hdfs quota
+    // is finished]] then fileTimestamp will be null, in that case check for max query out and
+    // delete the .write file after timeout
+    if (fileTimestamp == null) {
+      String tableUpdateStatusFilename = invalidFile.getName();
+      if (tableUpdateStatusFilename.endsWith(".write")) {
+        long tableUpdateStatusFileTimeStamp = Long.parseLong(
+            CarbonTablePath.DataFileUtil.getTimeStampFromFileName(tableUpdateStatusFilename));
+        if (isMaxQueryTimeoutExceeded(tableUpdateStatusFileTimeStamp)) {
+          isDeleted = deleteInvalidFiles(invalidFile);
+        }
+      }
+    } else {
+      // if the timestamp of the file is more than the current time by query execution timeout.
+      // then delete that file.
+      if (CarbonUpdateUtil.isMaxQueryTimeoutExceeded(fileTimestamp) || forceDelete) {
+        isDeleted = deleteInvalidFiles(invalidFile);
       }
     }
-    return false;
+    return isDeleted;
+  }
+
+  private static boolean deleteInvalidFiles(CarbonFile invalidFile) {
+    boolean isDeleted;
+    try {
+      LOGGER.info("deleting the invalid file : " + invalidFile.getName());
+      CarbonUtil.deleteFoldersAndFiles(invalidFile);
+      isDeleted = true;
+    } catch (IOException | InterruptedException e) {
+      LOGGER.error("error in clean up of invalid files." + e.getMessage(), e);
+      isDeleted = false;
+    }
+    return isDeleted;
   }
 
   public static boolean isBlockInvalid(SegmentStatus blockStatus) {