You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/07/10 11:27:09 UTC
carbondata git commit: [CARBONDATA-2706][BloomDataMap] clear bloom
index files after segment is deleted
Repository: carbondata
Updated Branches:
refs/heads/master 0112ed096 -> d8562e5bd
[CARBONDATA-2706][BloomDataMap] clear bloom index files after segment is deleted
clear bloom index files after corresponding segment is deleted and
cleaned
This closes #2461
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d8562e5b
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d8562e5b
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d8562e5b
Branch: refs/heads/master
Commit: d8562e5bd2bb7ed2ecb6a56b905e6b219b539caf
Parents: 0112ed0
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Mon Jul 9 11:36:13 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Tue Jul 10 19:26:52 2018 +0800
----------------------------------------------------------------------
.../carbondata/core/datamap/TableDataMap.java | 8 +++++
.../core/datamap/dev/DataMapFactory.java | 5 ++++
.../blockletindex/BlockletDataMapFactory.java | 5 ++++
.../carbondata/core/util/DeleteLoadFolders.java | 28 +++++++++++++++++-
.../bloom/BloomCoarseGrainDataMapFactory.java | 27 +++++++++++------
.../examples/MinMaxIndexDataMapFactory.java | 5 ++++
.../lucene/LuceneDataMapFactoryBase.java | 30 +++++++++++--------
.../testsuite/datamap/CGDataMapTestCase.scala | 7 +++++
.../testsuite/datamap/DataMapWriterSuite.scala | 7 +++++
.../testsuite/datamap/FGDataMapTestCase.scala | 7 +++++
.../testsuite/datamap/TestDataMapStatus.scala | 7 +++++
.../TestInsertAndOtherCommandConcurrent.scala | 7 +++++
.../management/CarbonCleanFilesCommand.scala | 5 ++++
.../BloomCoarseGrainDataMapFunctionSuite.scala | 31 ++++++++++++++++++++
14 files changed, 157 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
index cb7ec03..89a4c86 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
@@ -208,6 +208,14 @@ public final class TableDataMap extends OperationEventListener {
}
/**
+ * delete only the datamaps of the segments
+ */
+ public void deleteDatamapData(List<Segment> segments) {
+ for (Segment segment: segments) {
+ dataMapFactory.deleteDatamapData(segment);
+ }
+ }
+ /**
* delete datamap data if any
*/
public void deleteDatamapData() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
index ff4ef72..b115462 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/dev/DataMapFactory.java
@@ -111,6 +111,11 @@ public abstract class DataMapFactory<T extends DataMap> {
public abstract DataMapLevel getDataMapLevel();
/**
+ * delete datamap data in the specified segment
+ */
+ public abstract void deleteDatamapData(Segment segment);
+
+ /**
* delete datamap data if any
*/
public abstract void deleteDatamapData();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
index 51e6e21..175a2a4 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMapFactory.java
@@ -330,6 +330,11 @@ public class BlockletDataMapFactory extends CoarseGrainDataMapFactory
return null;
}
+ @Override
+ public void deleteDatamapData(Segment segment) {
+
+ }
+
@Override public void deleteDatamapData() {
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/core/src/main/java/org/apache/carbondata/core/util/DeleteLoadFolders.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DeleteLoadFolders.java b/core/src/main/java/org/apache/carbondata/core/util/DeleteLoadFolders.java
index 25ba553..0433ba4 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DeleteLoadFolders.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DeleteLoadFolders.java
@@ -18,10 +18,14 @@
package org.apache.carbondata.core.util;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.datamap.DataMapStoreManager;
+import org.apache.carbondata.core.datamap.Segment;
+import org.apache.carbondata.core.datamap.TableDataMap;
import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
import org.apache.carbondata.core.datastore.impl.FileFactory;
@@ -31,6 +35,7 @@ import org.apache.carbondata.core.locks.ICarbonLock;
import org.apache.carbondata.core.locks.LockUsage;
import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.metadata.SegmentFileStore;
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable;
import org.apache.carbondata.core.mutate.CarbonUpdateUtil;
import org.apache.carbondata.core.statusmanager.LoadMetadataDetails;
import org.apache.carbondata.core.statusmanager.SegmentStatus;
@@ -85,7 +90,22 @@ public final class DeleteLoadFolders {
LoadMetadataDetails[] loadDetails,
boolean isForceDelete,
List<PartitionSpec> specs) {
- for (LoadMetadataDetails oneLoad : loadDetails) {
+ CarbonTable carbonTable = DataMapStoreManager.getInstance().getCarbonTable(
+ absoluteTableIdentifier);
+ List<TableDataMap> indexDataMaps = new ArrayList<>();
+ try {
+ for (TableDataMap dataMap : DataMapStoreManager.getInstance().getAllDataMap(carbonTable)) {
+ if (dataMap.getDataMapSchema().isIndexDataMap()) {
+ indexDataMaps.add(dataMap);
+ }
+ }
+ } catch (IOException e) {
+ LOGGER.warn(String.format(
+ "Failed to get datamaps for %s.%s, therefore the datamap files could not be cleaned.",
+ absoluteTableIdentifier.getDatabaseName(), absoluteTableIdentifier.getTableName()));
+ }
+
+ for (final LoadMetadataDetails oneLoad : loadDetails) {
if (checkIfLoadCanBeDeletedPhysically(oneLoad, isForceDelete)) {
try {
if (oneLoad.getSegmentFile() != null) {
@@ -135,6 +155,12 @@ public final class DeleteLoadFolders {
}
}
+ List<Segment> segments = new ArrayList<>(1);
+ for (TableDataMap dataMap : indexDataMaps) {
+ segments.clear();
+ segments.add(new Segment(oneLoad.getLoadName()));
+ dataMap.deleteDatamapData(segments);
+ }
} catch (IOException e) {
LOGGER.warn("Unable to delete the file as per delete command " + oneLoad.getLoadName());
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
index 57c645f..aa6c5e1 100644
--- a/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
+++ b/datamap/bloom/src/main/java/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFactory.java
@@ -345,22 +345,31 @@ public class BloomCoarseGrainDataMapFactory extends DataMapFactory<CoarseGrainDa
}
@Override
+ public void deleteDatamapData(Segment segment) {
+ try {
+ String segmentId = segment.getSegmentNo();
+ String datamapPath = CarbonTablePath
+ .getDataMapStorePath(getCarbonTable().getTablePath(), segmentId, dataMapName);
+ if (FileFactory.isFileExist(datamapPath)) {
+ CarbonFile file = FileFactory.getCarbonFile(datamapPath,
+ FileFactory.getFileType(datamapPath));
+ CarbonUtil.deleteFoldersAndFilesSilent(file);
+ }
+ } catch (IOException | InterruptedException ex) {
+ LOGGER.error("Failed to delete datamap for segment_" + segment.getSegmentNo());
+ }
+ }
+
+ @Override
public void deleteDatamapData() {
SegmentStatusManager ssm =
new SegmentStatusManager(getCarbonTable().getAbsoluteTableIdentifier());
try {
List<Segment> validSegments = ssm.getValidAndInvalidSegments().getValidSegments();
for (Segment segment : validSegments) {
- String segmentId = segment.getSegmentNo();
- String datamapPath = CarbonTablePath
- .getDataMapStorePath(getCarbonTable().getTablePath(), segmentId, dataMapName);
- if (FileFactory.isFileExist(datamapPath)) {
- CarbonFile file = FileFactory.getCarbonFile(datamapPath,
- FileFactory.getFileType(datamapPath));
- CarbonUtil.deleteFoldersAndFilesSilent(file);
- }
+ deleteDatamapData(segment);
}
- } catch (IOException | InterruptedException ex) {
+ } catch (IOException e) {
LOGGER.error("drop datamap failed, failed to delete datamap directory");
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java
----------------------------------------------------------------------
diff --git a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java
index bc65a93..506b91a 100644
--- a/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java
+++ b/datamap/examples/src/minmaxdatamap/main/java/org/apache/carbondata/datamap/examples/MinMaxIndexDataMapFactory.java
@@ -156,6 +156,11 @@ public class MinMaxIndexDataMapFactory extends CoarseGrainDataMapFactory {
return this.dataMapMeta;
}
+ @Override
+ public void deleteDatamapData(Segment segment) {
+
+ }
+
@Override public void deleteDatamapData() {
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
index 5393746..8186841 100644
--- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
+++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapFactoryBase.java
@@ -181,19 +181,9 @@ abstract class LuceneDataMapFactoryBase<T extends DataMap> extends DataMapFactor
try {
List<Segment> validSegments = ssm.getValidAndInvalidSegments().getValidSegments();
for (Segment segment : validSegments) {
- String segmentId = segment.getSegmentNo();
- String datamapPath = CarbonTablePath
- .getDataMapStorePath(tableIdentifier.getTablePath(), segmentId, dataMapName);
- if (FileFactory.isFileExist(datamapPath)) {
- CarbonFile file =
- FileFactory.getCarbonFile(datamapPath, FileFactory.getFileType(datamapPath));
- CarbonUtil.deleteFoldersAndFilesSilent(file);
- }
+ deleteDatamapData(segment);
}
- } catch (IOException ex) {
- throw new MalformedDataMapCommandException(
- "drop datamap failed, failed to delete datamap directory");
- } catch (InterruptedException ex) {
+ } catch (IOException | RuntimeException ex) {
throw new MalformedDataMapCommandException(
"drop datamap failed, failed to delete datamap directory");
}
@@ -270,6 +260,22 @@ abstract class LuceneDataMapFactoryBase<T extends DataMap> extends DataMapFactor
}
+ @Override public void deleteDatamapData(Segment segment) {
+ try {
+ String segmentId = segment.getSegmentNo();
+ String datamapPath = CarbonTablePath
+ .getDataMapStorePath(tableIdentifier.getTablePath(), segmentId, dataMapName);
+ if (FileFactory.isFileExist(datamapPath)) {
+ CarbonFile file = FileFactory.getCarbonFile(datamapPath,
+ FileFactory.getFileType(datamapPath));
+ CarbonUtil.deleteFoldersAndFilesSilent(file);
+ }
+ } catch (IOException | InterruptedException ex) {
+ throw new RuntimeException(
+ "drop datamap failed, failed to delete datamap directory");
+ }
+ }
+
@Override public void deleteDatamapData() {
try {
deleteDatamap();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
index bc8b28b..9a7cb99 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
@@ -136,6 +136,13 @@ class CGDataMapFactory(
}
/**
+ * delete datamap of the segment
+ */
+ override def deleteDatamapData(segment: Segment): Unit = {
+ ???
+ }
+
+ /**
* delete datamap data if any
*/
override def deleteDatamapData(): Unit = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
index d4d78c9..434c363 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/DataMapWriterSuite.scala
@@ -71,6 +71,13 @@ class C2DataMapFactory(
}
/**
+ * delete datamap of the segment
+ */
+ override def deleteDatamapData(segment: Segment): Unit = {
+ ???
+ }
+
+ /**
* delete datamap data if any
*/
override def deleteDatamapData(): Unit = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
index 02ada2e..f4c12f7 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
@@ -129,6 +129,13 @@ class FGDataMapFactory(carbonTable: CarbonTable,
}
/**
+ * delete datamap of the segment
+ */
+ override def deleteDatamapData(segment: Segment): Unit = {
+ ???
+ }
+
+ /**
* delete datamap data if any
*/
override def deleteDatamapData(): Unit = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapStatus.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapStatus.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapStatus.scala
index c615078..0638a07 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapStatus.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapStatus.scala
@@ -267,6 +267,13 @@ class TestDataMapFactory(
}
/**
+ * delete datamap of the segment
+ */
+ override def deleteDatamapData(segment: Segment): Unit = {
+ ???
+ }
+
+ /**
* delete datamap data if any
*/
override def deleteDatamapData(): Unit = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala
index 2441b53..dbbcd4d 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/TestInsertAndOtherCommandConcurrent.scala
@@ -340,6 +340,13 @@ class WaitingDataMapFactory(
}
/**
+ * delete datamap of the segment
+ */
+ override def deleteDatamapData(segment: Segment): Unit = {
+
+ }
+
+ /**
* delete datamap data if any
*/
override def deleteDatamapData(): Unit = {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
index 1c4d068..9d0f149 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonCleanFilesCommand.scala
@@ -29,6 +29,7 @@ import org.apache.carbondata.api.CarbonStore
import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.datamap.DataMapStoreManager
import org.apache.carbondata.core.exception.ConcurrentOperationException
import org.apache.carbondata.core.indexstore.PartitionSpec
import org.apache.carbondata.core.metadata.schema.table.CarbonTable
@@ -57,6 +58,10 @@ case class CarbonCleanFilesCommand(
override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
carbonTable = CarbonEnv.getCarbonTable(databaseNameOp, tableName.get)(sparkSession)
+ val dms = carbonTable.getTableInfo.getDataMapSchemaList.asScala.map(_.getDataMapName)
+ val indexDms = DataMapStoreManager.getInstance.getAllDataMap(carbonTable).asScala
+ .filter(_.getDataMapSchema.isIndexDataMap)
+
if (carbonTable.hasAggregationDataMap) {
cleanFileCommands = carbonTable.getTableInfo.getDataMapSchemaList.asScala.map {
dataMapSchema =>
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d8562e5b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala
index d0e2cbd..92943b2 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala
@@ -2,12 +2,16 @@ package org.apache.carbondata.datamap.bloom
import java.io.File
+import org.apache.commons.io.FileUtils
+import org.apache.spark.sql.CarbonEnv
+import org.apache.spark.sql.test.Spark2TestQueryExecutor
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.datamap.status.DataMapStatusManager
import org.apache.carbondata.core.util.CarbonProperties
+import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.datamap.bloom.BloomCoarseGrainDataMapTestUtil.deleteFile
import org.apache.carbondata.datamap.bloom.BloomCoarseGrainDataMapTestUtil.createFile
import org.apache.carbondata.datamap.bloom.BloomCoarseGrainDataMapTestUtil.checkBasicQuery
@@ -759,6 +763,33 @@ class BloomCoarseGrainDataMapFunctionSuite extends QueryTest with BeforeAndAfte
sql(s"SELECT * FROM $normalTable WHERE isNull(c2)"))
}
+ test("test bloom datamap: deleting & clearning segment will clear datamap files") {
+ sql(s"CREATE TABLE $bloomDMSampleTable(c1 string, c2 int, c3 string) STORED BY 'carbondata'")
+ sql(
+ s"""
+ | CREATE DATAMAP $dataMapName on table $bloomDMSampleTable
+ | using 'bloomfilter'
+ | DMPROPERTIES('index_columns'='c1, c2')
+ """.stripMargin)
+ sql(s"INSERT INTO $bloomDMSampleTable SELECT 'c1v1', 1, 'c3v1'")
+ sql(s"INSERT INTO $bloomDMSampleTable SELECT 'c1v2', 2, 'c3v2'")
+
+ // two segments both has datamap files
+ val carbonTable = CarbonEnv.getCarbonTable(Option("default"), bloomDMSampleTable)(Spark2TestQueryExecutor.spark)
+ import scala.collection.JavaConverters._
+ (0 to 1).foreach { segId =>
+ val datamapPath = CarbonTablePath.getDataMapStorePath(carbonTable.getTablePath, segId.toString, dataMapName)
+ assert(FileUtils.listFiles(FileUtils.getFile(datamapPath), Array("bloomindex"), true).asScala.nonEmpty)
+ }
+ // delete and clean the first segment, the corresponding datamap files should be cleaned too
+ sql(s"DELETE FROM TABLE $bloomDMSampleTable WHERE SEGMENT.ID IN (0)")
+ sql(s"CLEAN FILES FOR TABLE $bloomDMSampleTable")
+ var datamapPath = CarbonTablePath.getDataMapStorePath(carbonTable.getTablePath, "0", dataMapName)
+ assert(!FileUtils.getFile(datamapPath).exists(), "index file of this segment has been deleted, should not exist")
+ datamapPath = CarbonTablePath.getDataMapStorePath(carbonTable.getTablePath, "1", dataMapName)
+ assert(FileUtils.listFiles(FileUtils.getFile(datamapPath), Array("bloomindex"), true).asScala.nonEmpty)
+ }
+
override def afterAll(): Unit = {
deleteFile(bigFile)
sql(s"DROP TABLE IF EXISTS $normalTable")