You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/03/30 06:05:00 UTC
carbondata git commit: [CARBONDATA-2289] If carbon merge index is
enabled then after IUD operation
Repository: carbondata
Updated Branches:
refs/heads/master 7e0803fec -> 5daae9515
[CARBONDATA-2289] If carbon merge index is enabled then after IUD operation
if some blocks of a segment is deleted, then during query and IUD operation the driver is throwing FileNotFoundException while preparing BlockMetaInfo.
Solution:
Skiping the invalid(deleted) blocks and logig the same as warning.
This closes #2110
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5daae951
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5daae951
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5daae951
Branch: refs/heads/master
Commit: 5daae951500967a910b863e4e57742b51f63f591
Parents: 7e0803f
Author: mohammadshahidkhan <mo...@gmail.com>
Authored: Wed Mar 28 16:38:21 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Fri Mar 30 11:34:27 2018 +0530
----------------------------------------------------------------------
.../core/indexstore/BlockletDataMapIndexStore.java | 8 +++++++-
.../datacompaction/CarbonIndexFileMergeTestCase.scala | 11 +++++++++++
2 files changed, 18 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5daae951/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
index befa121..167a04e 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
@@ -118,7 +118,13 @@ public class BlockletDataMapIndexStore
.getIndexFileName(), indexFileStore.getFileData(identifier.getIndexFileName()));
for (DataFileFooter footer : indexInfo) {
String blockPath = footer.getBlockInfo().getTableBlockInfo().getFilePath();
- blockMetaInfoMap.put(blockPath, createBlockMetaInfo(blockPath));
+ if (FileFactory.isFileExist(blockPath)) {
+ blockMetaInfoMap.put(blockPath, createBlockMetaInfo(blockPath));
+ } else {
+ LOGGER.warn("Skipping invalid block " + footer.getBlockInfo().getBlockUniqueName()
+ + " The block does not exist. The block might be got deleted due to clean up post"
+ + " update/delete operation over table.");
+ }
}
return blockMetaInfoMap;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/5daae951/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala
index c7912cf..9cb30b9 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala
@@ -178,6 +178,17 @@ class CarbonIndexFileMergeTestCase
checkAnswer(sql("""Select count(*) from nonindexmerge"""), rows)
}
+ test("Query should not fail after iud operation on a table having merge indexes") {
+ sql("drop table if exists mitable")
+ sql("create table mitable(id int, issue date) stored by 'carbondata'")
+ sql("insert into table mitable select '1','2000-02-01'")
+ val table = CarbonMetadata.getInstance().getCarbonTable("default", "mitable")
+ new CarbonIndexFileMergeWriter()
+ .mergeCarbonIndexFilesOfSegment("0", table.getTablePath, false)
+ sql("update mitable set(id)=(2) where issue = '2000-02-01'").show()
+ sql("clean files for table mitable")
+ sql("select * from mitable").show()
+ }
private def getIndexFileCount(tableName: String, segment: String): Int = {
val table = CarbonMetadata.getInstance().getCarbonTable(tableName)
val path = CarbonTablePath