You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/08/09 18:25:52 UTC
[08/47] carbondata git commit: [CARBONDATA-2805] Fix the ordering
mismatch of segment numbers during cutom compaction
[CARBONDATA-2805] Fix the ordering mismatch of segment numbers during cutom compaction
Problem:
when we have segments from 0 to 6 and i give 1, 2, 3 for custom compaction, then it should create 1.1 as compacted segment, but sometimes
it will create 3.1 as compacted segment which is wrong. This is beacuse custom Segment IDs were passing in hashset and finally inserted in
hashmap, while identifying segments to be merged. hashmap and hashset does not guarantee the insertion order which may lead to missmatch of segment numbers.
Solution:
Use LinkedHashSet and LinkedHashMap which always sure about the insertion order.
This closes #2585
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/259868cd
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/259868cd
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/259868cd
Branch: refs/heads/branch-1.4
Commit: 259868cd26780d5d63bdaefb0d0587787cbe6147
Parents: 2846edd
Author: akashrn5 <ak...@gmail.com>
Authored: Mon Jul 30 19:22:29 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Aug 9 23:38:51 2018 +0530
----------------------------------------------------------------------
.../processing/merger/CarbonDataMergerUtil.java | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/259868cd/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
index 78af751..1162fc2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java
@@ -391,7 +391,6 @@ public final class CarbonDataMergerUtil {
CarbonLoadModel carbonLoadModel, long compactionSize,
List<LoadMetadataDetails> segments, CompactionType compactionType,
List<String> customSegmentIds) throws IOException, MalformedCarbonCommandException {
- String tablePath = carbonLoadModel.getTablePath();
Map<String, String> tableLevelProperties = carbonLoadModel.getCarbonDataLoadSchema()
.getCarbonTable().getTableInfo().getFactTable().getTableProperties();
List<LoadMetadataDetails> sortedSegments = new ArrayList<LoadMetadataDetails>(segments);
@@ -400,7 +399,7 @@ public final class CarbonDataMergerUtil {
if (CompactionType.CUSTOM == compactionType) {
return identitySegmentsToBeMergedBasedOnSpecifiedSegments(sortedSegments,
- new HashSet<>(customSegmentIds));
+ new LinkedHashSet<>(customSegmentIds));
}
// Check for segments which are qualified for IUD compaction.
@@ -424,7 +423,7 @@ public final class CarbonDataMergerUtil {
if (CompactionType.MAJOR == compactionType) {
listOfSegmentsToBeMerged = identifySegmentsToBeMergedBasedOnSize(compactionSize,
- listOfSegmentsLoadedInSameDateInterval, carbonLoadModel, tablePath);
+ listOfSegmentsLoadedInSameDateInterval, carbonLoadModel);
} else {
listOfSegmentsToBeMerged =
@@ -462,7 +461,7 @@ public final class CarbonDataMergerUtil {
List<LoadMetadataDetails> listOfSegments,
Set<String> segmentIds) throws MalformedCarbonCommandException {
Map<String, LoadMetadataDetails> specifiedSegments =
- new HashMap<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+ new LinkedHashMap<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
for (LoadMetadataDetails detail : listOfSegments) {
if (segmentIds.contains(detail.getLoadName())) {
specifiedSegments.put(detail.getLoadName(), detail);
@@ -623,13 +622,12 @@ public final class CarbonDataMergerUtil {
* @param listOfSegmentsAfterPreserve the segments list after
* preserving the configured number of latest loads
* @param carbonLoadModel carbon load model
- * @param tablePath the store location of the segment
* @return the list of segments that need to be merged
* based on the Size in case of Major compaction
*/
private static List<LoadMetadataDetails> identifySegmentsToBeMergedBasedOnSize(
long compactionSize, List<LoadMetadataDetails> listOfSegmentsAfterPreserve,
- CarbonLoadModel carbonLoadModel, String tablePath) throws IOException {
+ CarbonLoadModel carbonLoadModel) throws IOException {
List<LoadMetadataDetails> segmentsToBeMerged =
new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);