You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by xu...@apache.org on 2018/07/18 01:12:19 UTC
carbondata git commit: [CARBONDATA-2746][BloomDataMap] Fix bug for
getting datamap file when table has multiple datamaps
Repository: carbondata
Updated Branches:
refs/heads/master a16289786 -> 4612e0031
[CARBONDATA-2746][BloomDataMap] Fix bug for getting datamap file when table has multiple datamaps
Currently, if table has multiple bloom datamap and carbon is set to use distributed datamap, query will throw an exception when accessing the index file, because carbon gets all the datamaps but sets them with same datamap schema. The error is appeared when getting the full path of bloom index by concating index directory and index column. This PR fix this problem by filter the index directories of target datamap when using distributed datamap.
Test shows that lucene is not affected by this. On the other hand, lucene gets wrong result if we apply this filter
This closes #2512
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4612e003
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4612e003
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4612e003
Branch: refs/heads/master
Commit: 4612e003186ccc6bae89443043bd0db3463f8fc1
Parents: a162897
Author: Manhua <ke...@qq.com>
Authored: Mon Jul 16 19:29:07 2018 +0800
Committer: xuchuanyin <xu...@hust.edu.cn>
Committed: Wed Jul 18 09:10:22 2018 +0800
----------------------------------------------------------------------
.../bloom/BloomCoarseGrainDataMapFactory.java | 27 +++++++------
.../lucene/LuceneFineGrainDataMapSuite.scala | 7 ++++
.../bloom/BloomCoarseGrainDataMapSuite.scala | 40 ++++++++++++++++++++
3 files changed, 62 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4612e003/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
index 35ebd20..4b5bc7c 100644
--- a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
+++ b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
@@ -278,18 +278,21 @@ public class BloomCoarseGrainDataMapFactory extends DataMapFactory<CoarseGrainDa
}
if (dataMaps.size() > 0) {
for (TableDataMap dataMap : dataMaps) {
- List<CarbonFile> indexFiles;
- String dmPath = CarbonTablePath
- .getDataMapStorePath(tablePath, segmentId, dataMap.getDataMapSchema().getDataMapName());
- FileFactory.FileType fileType = FileFactory.getFileType(dmPath);
- final CarbonFile dirPath = FileFactory.getCarbonFile(dmPath, fileType);
- indexFiles = Arrays.asList(dirPath.listFiles(new CarbonFileFilter() {
- @Override
- public boolean accept(CarbonFile file) {
- return file.isDirectory();
- }
- }));
- indexDirs.addAll(indexFiles);
+ // different from lucene, bloom only get corresponding directory of current datamap
+ if (dataMap.getDataMapSchema().getDataMapName().equals(this.dataMapName)) {
+ List<CarbonFile> indexFiles;
+ String dmPath = CarbonTablePath.getDataMapStorePath(tablePath, segmentId,
+ dataMap.getDataMapSchema().getDataMapName());
+ FileFactory.FileType fileType = FileFactory.getFileType(dmPath);
+ final CarbonFile dirPath = FileFactory.getCarbonFile(dmPath, fileType);
+ indexFiles = Arrays.asList(dirPath.listFiles(new CarbonFileFilter() {
+ @Override
+ public boolean accept(CarbonFile file) {
+ return file.isDirectory();
+ }
+ }));
+ indexDirs.addAll(indexFiles);
+ }
}
}
return indexDirs.toArray(new CarbonFile[0]);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4612e003/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
index 657a3eb..aebbde4 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
@@ -34,6 +34,10 @@ import org.apache.carbondata.core.datamap.status.DataMapStatusManager
class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll {
+ val originDistributedDatamapStatus = CarbonProperties.getInstance().getProperty(
+ CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP,
+ CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP_DEFAULT
+ )
val file2 = resourcesPath + "/datamap_input.csv"
override protected def beforeAll(): Unit = {
@@ -908,6 +912,9 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll {
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_LUCENE_INDEX_STOP_WORDS,
CarbonCommonConstants.CARBON_LUCENE_INDEX_STOP_WORDS_DEFAULT)
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP,
+ originDistributedDatamapStatus)
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/4612e003/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
index 0b0c665..12cd234 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
@@ -415,6 +415,46 @@ class BloomCoarseGrainDataMapSuite extends QueryTest with BeforeAndAfterAll with
checkQuery("fakeDm", shouldHit = false)
}
+ test("test create datamaps on different column but hit only one") {
+ val originDistributedDatamapStatus = CarbonProperties.getInstance().getProperty(
+ CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP,
+ CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP_DEFAULT
+ )
+
+ CarbonProperties.getInstance()
+ .addProperty(CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP, "true")
+ val datamap1 = "datamap1"
+ val datamap2 = "datamap2"
+ sql(
+ s"""
+ | CREATE TABLE $bloomDMSampleTable(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'carbondata'
+ | """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP $datamap1 ON TABLE $bloomDMSampleTable
+ | USING 'bloomfilter'
+ | DMProperties('INDEX_COLUMNS'='name', 'BLOOM_SIZE'='64000', 'BLOOM_FPP'='0.00001')
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP $datamap2 ON TABLE $bloomDMSampleTable
+ | USING 'bloomfilter'
+ | DMProperties('INDEX_COLUMNS'='city', 'BLOOM_SIZE'='64000', 'BLOOM_FPP'='0.00001')
+ """.stripMargin)
+
+ sql(
+ s"""
+ | INSERT INTO $bloomDMSampleTable
+ | VALUES(5,'a','beijing',21),(6,'b','shanghai',25),(7,'b','guangzhou',28)
+ """.stripMargin)
+ assert(sql(s"SELECT * FROM $bloomDMSampleTable WHERE city='shanghai'").count() == 1)
+
+ // recover original setting
+ CarbonProperties.getInstance().addProperty(CarbonCommonConstants.USE_DISTRIBUTED_DATAMAP,
+ originDistributedDatamapStatus)
+ }
+
test("test block change datatype for bloomfilter index datamap") {
sql(
s"""