You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ma...@apache.org on 2018/03/30 14:17:18 UTC
carbondata git commit: [CARBONDATA-2289] If carbon merge index is
enabled then after IUD operation if some blocks of a segment is deleted,
then during query and IUD operation the driver is throwing
FileNotFoundException while preparing BlockMetaInfo.
Repository: carbondata
Updated Branches:
refs/heads/branch-1.3 b835e76a7 -> 97e7eca64
[CARBONDATA-2289] If carbon merge index is enabled then after IUD operation if some blocks of a segment is deleted, then during query and
IUD operation the driver is throwing FileNotFoundException while preparing BlockMetaInfo.
Solution:
Skiping the invalid(deleted) blocks and logig the same as warning.
This closes #2118
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/97e7eca6
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/97e7eca6
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/97e7eca6
Branch: refs/heads/branch-1.3
Commit: 97e7eca64dd2fb85839886ffd8b07c2bbce5e5b3
Parents: b835e76
Author: mohammadshahidkhan <mo...@gmail.com>
Authored: Wed Mar 28 16:38:21 2018 +0530
Committer: manishgupta88 <to...@gmail.com>
Committed: Fri Mar 30 19:49:51 2018 +0530
----------------------------------------------------------------------
.../core/indexstore/BlockletDataMapIndexStore.java | 8 +++++++-
.../datacompaction/CarbonIndexFileMergeTestCase.scala | 11 +++++++++++
2 files changed, 18 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/97e7eca6/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
index 75799b3..ac14105 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
@@ -119,7 +119,13 @@ public class BlockletDataMapIndexStore
.getIndexFileName(), indexFileStore.getFileData(identifier.getIndexFileName()));
for (DataFileFooter footer : indexInfo) {
String blockPath = footer.getBlockInfo().getTableBlockInfo().getFilePath();
- blockMetaInfoMap.put(blockPath, createBlockMetaInfo(blockPath));
+ if (FileFactory.isFileExist(blockPath)) {
+ blockMetaInfoMap.put(blockPath, createBlockMetaInfo(blockPath));
+ } else {
+ LOGGER.warn("Skipping invalid block " + footer.getBlockInfo().getBlockUniqueName()
+ + " The block does not exist. The block might be got deleted due to clean up post"
+ + " update/delete operation over table.");
+ }
}
return blockMetaInfoMap;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/97e7eca6/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala
index 7608318..e2e40e5 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala
@@ -182,6 +182,17 @@ class CarbonIndexFileMergeTestCase
checkAnswer(sql("""Select count(*) from nonindexmerge"""), rows)
}
+ test("Query should not fail after iud operation on a table having merge indexes") {
+ sql("drop table if exists mitable")
+ sql("create table mitable(id int, issue date) stored by 'carbondata'")
+ sql("insert into table mitable select '1','2000-02-01'")
+ val table = CarbonMetadata.getInstance().getCarbonTable("default", "mitable")
+ new CarbonIndexFileMergeWriter()
+ .mergeCarbonIndexFilesOfSegment("0", table.getTablePath, false)
+ sql("update mitable set(id)=(2) where issue = '2000-02-01'").show()
+ sql("clean files for table mitable")
+ sql("select * from mitable").show()
+ }
private def getIndexFileCount(tableName: String, segment: String): Int = {
val table = CarbonMetadata.getInstance().getCarbonTable(tableName)
val path = CarbonTablePath