You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/08/08 06:25:44 UTC
carbondata git commit: [CARBONDATA-2834] Remove unnecessary nested
looping over loadMetadatadetails.
Repository: carbondata
Updated Branches:
refs/heads/master 8e54f1e45 -> 01115872e
[CARBONDATA-2834] Remove unnecessary nested looping over loadMetadatadetails.
removed nested for loop which causes query performance degradation if there are huge number of entries for compacted segments in tablestatus file.
This closes #2612
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/01115872
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/01115872
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/01115872
Branch: refs/heads/master
Commit: 01115872efc5eaa87886ca53f85dc680752fb750
Parents: 8e54f1e
Author: kunal642 <ku...@gmail.com>
Authored: Tue Aug 7 15:12:01 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Wed Aug 8 11:55:27 2018 +0530
----------------------------------------------------------------------
.../SegmentUpdateStatusManager.java | 27 ++++++++++++++++++++
.../hadoop/api/CarbonTableInputFormat.java | 5 +---
2 files changed, 28 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/01115872/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
index c3daac5..c340714 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
@@ -779,6 +779,33 @@ public class SegmentUpdateStatusManager {
}
return range;
}
+
+ /**
+ * Returns the invalid timestamp range of a segment.
+ * @return
+ */
+ public List<UpdateVO> getInvalidTimestampRange() {
+ List<UpdateVO> ranges = new ArrayList<UpdateVO>();
+ for (LoadMetadataDetails segment : segmentDetails) {
+ if ((SegmentStatus.LOAD_FAILURE == segment.getSegmentStatus()
+ || SegmentStatus.COMPACTED == segment.getSegmentStatus()
+ || SegmentStatus.MARKED_FOR_DELETE == segment.getSegmentStatus())) {
+ UpdateVO range = new UpdateVO();
+ range.setSegmentId(segment.getLoadName());
+ range.setFactTimestamp(segment.getLoadStartTime());
+ if (!segment.getUpdateDeltaStartTimestamp().isEmpty() &&
+ !segment.getUpdateDeltaEndTimestamp().isEmpty()) {
+ range.setUpdateDeltaStartTimestamp(
+ CarbonUpdateUtil.getTimeStampAsLong(segment.getUpdateDeltaStartTimestamp()));
+ range.setLatestUpdateTimestamp(
+ CarbonUpdateUtil.getTimeStampAsLong(segment.getUpdateDeltaEndTimestamp()));
+ }
+ ranges.add(range);
+ }
+ }
+ return ranges;
+ }
+
/**
*
* @param block
http://git-wip-us.apache.org/repos/asf/carbondata/blob/01115872/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
index be72983..d88ca88 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/api/CarbonTableInputFormat.java
@@ -180,10 +180,7 @@ public class CarbonTableInputFormat<T> extends CarbonInputFormat<T> {
}
// remove entry in the segment index if there are invalid segments
invalidSegments.addAll(segments.getInvalidSegments());
- for (Segment invalidSegmentId : invalidSegments) {
- invalidTimestampsList
- .add(updateStatusManager.getInvalidTimestampRange(invalidSegmentId.getSegmentNo()));
- }
+ invalidTimestampsList.addAll(updateStatusManager.getInvalidTimestampRange());
if (invalidSegments.size() > 0) {
DataMapStoreManager.getInstance()
.clearInvalidSegments(getOrCreateCarbonTable(job.getConfiguration()), invalidSegments);