[carbondata] branch master updated: [CARBONDATA-3878] Get last modified time from 'tablestatus' file entry instead of segment file to reduce file operation 'getLastModifiedTime'

[li...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

liuzhi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git


The following commit(s) were added to refs/heads/master by this push:
     new 16da96c  [CARBONDATA-3878] Get last modified time from 'tablestatus' file entry instead of segment file to reduce file operation 'getLastModifiedTime'
16da96c is described below

commit 16da96c9a1531dce1031f4702166270eb9d3fb90
Author: QiangCai <qi...@qq.com>
AuthorDate: Sun Jun 28 20:12:12 2020 +0800

    [CARBONDATA-3878] Get last modified time from 'tablestatus' file entry instead of segment file to reduce file operation 'getLastModifiedTime'
    
    Why is this PR needed?
    After the table has too many segments, file operation 'getLastModifiedTime' on all segment files will take a long time.
    
    What changes were proposed in this PR?
    Get last modified time from 'tablestatus' file entry instead of segment file to reduce file operation 'getLastModifiedTime'
    
    Does this PR introduce any user interface change?
    No
    
    Is any new testcase added?
    No
    
    This closes #3814
---
 .../org/apache/carbondata/core/index/IndexStoreManager.java | 13 +++++++++----
 .../core/readcommitter/TableStatusReadCommittedScope.java   |  4 +++-
 .../carbondata/core/statusmanager/LoadMetadataDetails.java  |  7 +++++++
 3 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/core/src/main/java/org/apache/carbondata/core/index/IndexStoreManager.java b/core/src/main/java/org/apache/carbondata/core/index/IndexStoreManager.java
index 106042c..f38c33f 100644
--- a/core/src/main/java/org/apache/carbondata/core/index/IndexStoreManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/index/IndexStoreManager.java
@@ -515,11 +515,16 @@ public final class IndexStoreManager {
         UpdateVO updateVO =
             SegmentUpdateStatusManager.getInvalidTimestampRange(segment.getLoadMetadataDetails());
         SegmentRefreshInfo segmentRefreshInfo;
-        if (updateVO != null && updateVO.getLatestUpdateTimestamp() != null
+        if ((updateVO != null && updateVO.getLatestUpdateTimestamp() != null)
             || segment.getSegmentFileName() != null) {
-          long segmentFileTimeStamp = FileFactory.getCarbonFile(CarbonTablePath
-              .getSegmentFilePath(table.getTablePath(), segment.getSegmentFileName()))
-              .getLastModifiedTime();
+          long segmentFileTimeStamp;
+          if (null != segment.getLoadMetadataDetails()) {
+            segmentFileTimeStamp = segment.getLoadMetadataDetails().getLastModifiedTime();
+          } else {
+            segmentFileTimeStamp = FileFactory.getCarbonFile(CarbonTablePath
+                .getSegmentFilePath(table.getTablePath(), segment.getSegmentFileName()))
+                .getLastModifiedTime();
+          }
           segmentRefreshInfo =
               new SegmentRefreshInfo(updateVO.getLatestUpdateTimestamp(), 0, segmentFileTimeStamp);
         } else {
diff --git a/core/src/main/java/org/apache/carbondata/core/readcommitter/TableStatusReadCommittedScope.java b/core/src/main/java/org/apache/carbondata/core/readcommitter/TableStatusReadCommittedScope.java
index d6e3a62..4c73833 100644
--- a/core/src/main/java/org/apache/carbondata/core/readcommitter/TableStatusReadCommittedScope.java
+++ b/core/src/main/java/org/apache/carbondata/core/readcommitter/TableStatusReadCommittedScope.java
@@ -95,7 +95,9 @@ public class TableStatusReadCommittedScope implements ReadCommittedScope {
   public SegmentRefreshInfo getCommittedSegmentRefreshInfo(Segment segment, UpdateVO updateVo) {
     SegmentRefreshInfo segmentRefreshInfo;
     long segmentFileTimeStamp = 0L;
-    if (null != segment.getSegmentFileName()) {
+    if (null != segment.getLoadMetadataDetails()) {
+      segmentFileTimeStamp = segment.getLoadMetadataDetails().getLastModifiedTime();
+    } else if (null != segment.getSegmentFileName()) {
       segmentFileTimeStamp = FileFactory.getCarbonFile(CarbonTablePath
           .getSegmentFilePath(identifier.getTablePath(), segment.getSegmentFileName()))
           .getLastModifiedTime();
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
index 0eef6af..7fb2cbd 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/LoadMetadataDetails.java
@@ -487,4 +487,11 @@ public class LoadMetadataDetails implements Serializable {
       fileFormat = null;
     }
   }
+
+  public long getLastModifiedTime() {
+    if (updateDeltaEndTimestamp != null) {
+      return convertTimeStampToLong(updateDeltaEndTimestamp);
+    }
+    return convertTimeStampToLong(timestamp);
+  }
 }