You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/06/30 06:25:22 UTC
carbondata git commit: [CARBONDATA-2635][BloomDataMap] Support
different index datamaps on same column
Repository: carbondata
Updated Branches:
refs/heads/master 589fe1883 -> 00d6ce49d
[CARBONDATA-2635][BloomDataMap] Support different index datamaps on same column
User can create different provider based index datamaps on one column,
for example user can create bloomfilter datamap and lucene datamap on
one column, but not able to create two bloomfilter datamap on one
column.
This closes #2405
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/00d6ce49
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/00d6ce49
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/00d6ce49
Branch: refs/heads/master
Commit: 00d6ce49d924bf58d4480e61cd0b64d3c5e54181
Parents: 589fe18
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Sun Jun 24 12:47:21 2018 +0800
Committer: Jacky Li <ja...@qq.com>
Committed: Sat Jun 30 14:25:07 2018 +0800
----------------------------------------------------------------------
.../core/datamap/DataMapProvider.java | 2 +-
.../lucene/LuceneFineGrainDataMapSuite.scala | 26 ----------
.../testsuite/datamap/TestDataMapCommand.scala | 51 ++++++++++++++++++++
.../datamap/CarbonCreateDataMapCommand.scala | 20 ++++----
4 files changed, 63 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/00d6ce49/core/src/main/java/org/apache/carbondata/core/datamap/DataMapProvider.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapProvider.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapProvider.java
index 05ba7cf..168159b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapProvider.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapProvider.java
@@ -67,7 +67,7 @@ public abstract class DataMapProvider {
return mainTable;
}
- protected final DataMapSchema getDataMapSchema() {
+ public final DataMapSchema getDataMapSchema() {
return dataMapSchema;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/00d6ce49/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
index b3c1a1a..dc0cf52 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
@@ -225,32 +225,6 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll {
sql("DROP TABLE IF EXISTS datamap_test3")
}
- test("test lucene fine grain data map for create datamap with Duplicate Columns") {
- sql("DROP TABLE IF EXISTS datamap_test_table")
- sql(
- """
- | CREATE TABLE datamap_test_table(id INT, name STRING, city STRING, age INT)
- | STORED BY 'carbondata'
- | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
- """.stripMargin)
- val exception_duplicate_column: Exception = intercept[MalformedDataMapCommandException] {
- sql(
- s"""
- | CREATE DATAMAP dm ON TABLE datamap_test_table
- | USING 'lucene'
- | DMProperties('INDEX_COLUMNS'='name')
- """.stripMargin)
- sql(
- s"""
- | CREATE DATAMAP dm1 ON TABLE datamap_test_table
- | USING 'lucene'
- | DMProperties('INDEX_COLUMNS'='name')
- """.stripMargin)
- }
- assertResult("column 'name' already has datamap created")(exception_duplicate_column.getMessage)
- sql("drop datamap if exists dm on table datamap_test_table")
- }
-
test("test lucene fine grain data map with wildcard matching ") {
sql("DROP TABLE IF EXISTS datamap_test_table")
sql(
http://git-wip-us.apache.org/repos/asf/carbondata/blob/00d6ce49/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
index afca3b2..7fc4e26 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
@@ -350,6 +350,57 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
sql("drop table main")
}
+ test("test create datamap: unable to create same index datamap for one column") {
+ sql("DROP TABLE IF EXISTS datamap_test_table")
+ sql(
+ """
+ | CREATE TABLE datamap_test_table(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ val exception_duplicate_column: Exception = intercept[MalformedDataMapCommandException] {
+ sql(
+ s"""
+ | CREATE DATAMAP dm ON TABLE datamap_test_table
+ | USING 'lucene'
+ | DMProperties('INDEX_COLUMNS'='name')
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP dm1 ON TABLE datamap_test_table
+ | USING 'lucene'
+ | DMProperties('INDEX_COLUMNS'='name')
+ """.stripMargin)
+ }
+ assertResult("column 'name' already has lucene index datamap created")(exception_duplicate_column.getMessage)
+ sql("drop table if exists datamap_test_table")
+ }
+
+ test("test create datamap: able to create different index datamap for one column") {
+ sql("DROP TABLE IF EXISTS datamap_test_table")
+ sql(
+ """
+ | CREATE TABLE datamap_test_table(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'carbondata'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP dm ON TABLE datamap_test_table
+ | USING 'lucene'
+ | DMProperties('INDEX_COLUMNS'='name')
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP dm1 ON TABLE datamap_test_table
+ | USING 'bloomfilter'
+ | DMProperties('INDEX_COLUMNS'='name')
+ """.stripMargin)
+ sql("show datamap on table datamap_test_table").show(false)
+ checkExistence(sql("show datamap on table datamap_test_table"), true, "dm", "dm1", "lucene", "bloomfilter")
+ sql("drop table if exists datamap_test_table")
+ }
+
override def afterAll {
sql("DROP TABLE IF EXISTS maintable")
sql("drop table if exists uniqdata")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/00d6ce49/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
index 27e1720..a1a86a4 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
@@ -16,10 +16,7 @@
*/
package org.apache.spark.sql.execution.command.datamap
-import java.util
-
import scala.collection.JavaConverters._
-import scala.collection.mutable
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.TableIdentifier
@@ -99,17 +96,22 @@ case class CarbonCreateDataMapCommand(
dataMapProvider match {
case provider: IndexDataMapProvider =>
val datamaps = DataMapStoreManager.getInstance.getAllDataMap(mainTable).asScala
- val existingIndexColumn = mutable.Set[String]()
- datamaps.foreach { datamap =>
- datamap.getDataMapSchema.getIndexColumns.foreach(existingIndexColumn.add)
- }
+ val thisDmProviderName =
+ dataMapProvider.asInstanceOf[IndexDataMapProvider].getDataMapSchema.getProviderName
+ val existingIndexColumn4ThisProvider = datamaps.filter { datamap =>
+ thisDmProviderName.equalsIgnoreCase(datamap.getDataMapSchema.getProviderName)
+ }.flatMap { datamap =>
+ datamap.getDataMapSchema.getIndexColumns
+ }.distinct
provider.getIndexedColumns.asScala.foreach { column =>
- if (existingIndexColumn.contains(column.getColName)) {
+ if (existingIndexColumn4ThisProvider.contains(column.getColName)) {
throw new MalformedDataMapCommandException(String.format(
- "column '%s' already has datamap created", column.getColName))
+ "column '%s' already has %s index datamap created",
+ column.getColName, thisDmProviderName))
}
}
+
val operationContext: OperationContext = new OperationContext()
val systemFolderLocation: String = CarbonProperties.getInstance().getSystemFolderLocation
val createDataMapPreExecutionEvent: CreateDataMapPreExecutionEvent =