You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/08/09 18:26:03 UTC
[19/47] carbondata git commit: [CARBONDATA-2753][Compatibility] Merge
Index file not getting created with blocklet information for old store
[CARBONDATA-2753][Compatibility] Merge Index file not getting created with blocklet information for old store
Problem
Merge Index file not getting created with blocklet information for old store
Analysis
In legacy store (store <= 1.1 version), blocklet information is not written in the carbon Index files. When merge Index is created using the Alter DDL command on old store then merge Index file should be created with blocklet information which is as per the new store. This is not happening because the flag to read the carbondata file footer is not passed as true from Alter DDL command flow.
Fix
Pass the flag to read carbondataFileFooter as true while creating the merge Index file using Alter DDL command
This closes #2593
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/e183d713
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/e183d713
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/e183d713
Branch: refs/heads/branch-1.4
Commit: e183d71321a2fd0df1aa9f399e409b835bec6328
Parents: d0567af
Author: manishgupta88 <to...@gmail.com>
Authored: Wed Aug 1 14:24:52 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Thu Aug 9 23:42:44 2018 +0530
----------------------------------------------------------------------
.../blockletindex/SegmentIndexFileStore.java | 2 +-
.../sql/events/MergeIndexEventListener.scala | 18 ++++++++++++------
2 files changed, 13 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e183d713/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
index 16910ac..c4e7f7a 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/SegmentIndexFileStore.java
@@ -405,7 +405,7 @@ public class SegmentIndexFileStore {
// get the index header
org.apache.carbondata.format.IndexHeader indexHeader = indexReader.readIndexHeader();
DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
- String filePath = indexFile.getCanonicalPath();
+ String filePath = FileFactory.getUpdatedFilePath(indexFile.getCanonicalPath());
String parentPath =
filePath.substring(0, filePath.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR));
while (indexReader.hasNext()) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/e183d713/integration/spark2/src/main/scala/org/apache/spark/sql/events/MergeIndexEventListener.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/events/MergeIndexEventListener.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/events/MergeIndexEventListener.scala
index dff3424..5bff9aa 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/events/MergeIndexEventListener.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/events/MergeIndexEventListener.scala
@@ -118,12 +118,18 @@ class MergeIndexEventListener extends OperationEventListener with Logging {
.put(loadMetadataDetails.getLoadName,
String.valueOf(loadMetadataDetails.getLoadStartTime))
})
- CommonUtil.mergeIndexFiles(sparkSession.sparkContext,
- validSegmentIds,
- segmentFileNameMap,
- carbonMainTable.getTablePath,
- carbonMainTable,
- true)
+ // in case of merge index file creation using Alter DDL command
+ // readFileFooterFromCarbonDataFile flag should be true. This flag is check for legacy
+ // store (store <= 1.1 version) and create merge Index file as per new store so that
+ // old store is also upgraded to new store
+ CommonUtil.mergeIndexFiles(
+ sparkContext = sparkSession.sparkContext,
+ segmentIds = validSegmentIds,
+ segmentFileNameToSegmentIdMap = segmentFileNameMap,
+ tablePath = carbonMainTable.getTablePath,
+ carbonTable = carbonMainTable,
+ mergeIndexProperty = true,
+ readFileFooterFromCarbonDataFile = true)
// clear Block dataMap Cache
clearBlockDataMapCache(carbonMainTable, validSegmentIds)
val requestMessage = "Compaction request completed for table "