You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/08/29 06:43:07 UTC
carbondata git commit: [CARBONDATA-2819] Fixed cannot drop preagg
datamap on table which has other type datamaps
Repository: carbondata
Updated Branches:
refs/heads/master 4df5f2f1b -> d801548aa
[CARBONDATA-2819] Fixed cannot drop preagg datamap on table which has other type datamaps
As we know, carbon now write preagg datamap info to main table schema and write other type datamap info like bloom or lucene to .dmschema file in _system folder. When dropping datamap, we should both check the .dmschema files and main table info if has the datamap which to be deleted.
This closes #2641
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d801548a
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d801548a
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d801548a
Branch: refs/heads/master
Commit: d801548aa704347f0f76c07c5e8099fd31945768
Parents: 4df5f2f
Author: Sssan520 <li...@aliyun.com>
Authored: Mon Jul 2 19:12:24 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Wed Aug 29 14:42:49 2018 +0800
----------------------------------------------------------------------
.../preaggregate/TestPreAggregateDrop.scala | 32 ++++++++++++++++++++
.../datamap/CarbonDropDataMapCommand.scala | 18 ++++++-----
2 files changed, 43 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d801548a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateDrop.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateDrop.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateDrop.scala
index f73a587..f99841c 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateDrop.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateDrop.scala
@@ -129,6 +129,38 @@ class TestPreAggregateDrop extends QueryTest with BeforeAndAfterAll {
assert(e.getMessage.contains("Table or view 'maintable' not found in"))
}
+ test("drop preaggregate datamap whose main table has other type datamaps") {
+ sql("drop table if exists maintable")
+ sql("create table maintable (a string, b string, c string) stored by 'carbondata'")
+ sql(
+ "create datamap bloom1 on table maintable using 'bloomfilter'" +
+ " dmproperties('index_columns'='a')")
+ sql(
+ "create datamap preagg1 on table maintable using 'preaggregate' as select" +
+ " a,sum(b) from maintable group by a")
+ sql("drop datamap if exists bloom1 on table maintable")
+ sql("drop datamap if exists preagg1 on table maintable")
+ checkExistence(sql("SHOW DATAMAP ON TABLE maintable"), false, "preagg1", "bloom1")
+ sql(
+ "create datamap bloom1 on table maintable using 'bloomfilter'" +
+ " dmproperties('index_columns'='a')")
+ sql(
+ "create datamap preagg1 on table maintable using 'preaggregate' as select" +
+ " a,sum(b) from maintable group by a")
+ sql("drop datamap if exists preagg1 on table maintable")
+ sql("drop datamap if exists bloom1 on table maintable")
+ checkExistence(sql("SHOW DATAMAP ON TABLE maintable"), false, "preagg1", "bloom1")
+ sql(
+ "create datamap bloom1 on table maintable using 'bloomfilter'" +
+ " dmproperties('index_columns'='a')")
+ sql(
+ "create datamap preagg1 on table maintable using 'preaggregate' as select" +
+ " a,sum(b) from maintable group by a")
+ sql("drop table if exists maintable")
+ checkExistence(sql("show tables"), false, "maintable_preagg1", "maintable")
+ checkExistence(sql("show datamap"), false, "preagg1", "bloom1")
+ }
+
override def afterAll() {
sql("drop table if exists maintable")
sql("drop table if exists maintable1")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/d801548a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
index 8bc269b..4607de0 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonDropDataMapCommand.scala
@@ -113,13 +113,18 @@ case class CarbonDropDataMapCommand(
}
LOGGER.audit(s"Deleting datamap [$dataMapName] under table [$tableName]")
- // drop index datamap on the main table
+ // drop index,mv datamap on the main table.
if (mainTable != null &&
DataMapStoreManager.getInstance().getAllDataMap(mainTable).size() > 0) {
- dropDataMapFromSystemFolder(sparkSession)
- return Seq.empty
+ val isDMSchemaExists = DataMapStoreManager.getInstance().getAllDataMap(mainTable).asScala.
+ exists(_.getDataMapSchema.getDataMapName.equalsIgnoreCase(dataMapName))
+ if (isDMSchemaExists) {
+ dropDataMapFromSystemFolder(sparkSession)
+ return Seq.empty
+ }
}
+ // drop preaggregate datamap.
// If datamap to be dropped in parent table then drop the datamap from metastore and remove
// entry from parent table.
// If force drop is true then remove the datamap from hivemetastore. No need to remove from
@@ -160,9 +165,8 @@ case class CarbonDropDataMapCommand(
} else if (!ifExistsSet) {
throw new NoSuchDataMapException(dataMapName, tableName)
}
- } else if (mainTable != null &&
- mainTable.getTableInfo.getDataMapSchemaList.size() == 0) {
- dropDataMapFromSystemFolder(sparkSession)
+ } else if (!ifExistsSet) {
+ throw new NoSuchDataMapException(dataMapName)
}
} catch {
case e: NoSuchDataMapException =>
@@ -191,7 +195,7 @@ case class CarbonDropDataMapCommand(
}
}
- Seq.empty
+ Seq.empty
}
private def dropDataMapFromSystemFolder(sparkSession: SparkSession): Unit = {