You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by aj...@apache.org on 2019/12/11 01:36:46 UTC
[carbondata] branch master updated: [CARBONDATA-3351] Block bloom
datamap creation on binary datatype column
This is an automated email from the ASF dual-hosted git repository.
ajantha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 27a689a [CARBONDATA-3351] Block bloom datamap creation on binary datatype column
27a689a is described below
commit 27a689a12a0cfce7fb151ec3348e48b4793656e5
Author: akkio-97 <ak...@gmail.com>
AuthorDate: Thu Dec 5 20:22:39 2019 +0530
[CARBONDATA-3351] Block bloom datamap creation on binary datatype column
Why this PR?
As per Binary Datatype design Bloom data-map creation on Binary datatype
column is not supported.
Handled the same.
This closes #3500
---
.../testsuite/datamap/TestDataMapCommand.scala | 61 +---------------------
.../datamap/CarbonCreateDataMapCommand.scala | 7 +++
.../bloom/BloomCoarseGrainDataMapSuite.scala | 19 +++++++
3 files changed, 27 insertions(+), 60 deletions(-)
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
index d9461b2..0da9428 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala
@@ -266,66 +266,7 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll {
sql(s"drop table if exists $tableName")
}
- test("test support bloomFilter on binary data type") {
- val tableName = "datamapshowtest"
- val datamapName = "bloomdatamap"
- val datamapName2 = "bloomdatamap2"
- sql(s"drop table if exists $tableName")
- // for index datamap
- sql(s"create table $tableName (a string, b string, c string, d binary, e binary) stored by 'carbondata'")
-
- sql(s"insert into $tableName values('a1','b1','c1','d1','e1')")
- sql(s"insert into $tableName values('a1','b2','c2','d1','e2')")
- sql(s"insert into $tableName values('a3','b3','c1','d2','e2')")
- sql(
- s"""
- | create datamap $datamapName on table $tableName using 'bloomfilter'
- | DMPROPERTIES ('index_columns'='d', 'bloom_size'='32000', 'bloom_fpp'='0.001')
- """.stripMargin)
- sql(
- s"""
- | create datamap $datamapName2 on table $tableName using 'bloomfilter'
- | DMPROPERTIES ('index_columns'='e')
- """.stripMargin)
- var bloom1 = sql(s"select * from $tableName where d=cast('d1' as binary)")
- assert(2 == bloom1.collect().length)
- bloom1.collect().foreach { each =>
- assert(5 == each.length)
- assert("a1".equals(each.get(0)))
- assert("d1".equals(new String(each.getAs[Array[Byte]](3))))
- if ("b1".equals(each.get(1))) {
- assert("c1".equals(each.get(2)))
- assert("e1".equals(new String(each.getAs[Array[Byte]](4))))
- } else if ("b2".equals(each.get(1))) {
- assert("c2".equals(each.get(2)))
- assert("e2".equals(new String(each.getAs[Array[Byte]](4))))
- } else {
- assert(false)
- }
- }
-
- bloom1 = sql(s"select * from $tableName where d=cast('d1' as binary) and e=cast('e1' as binary)")
- assert(1 == bloom1.collect().length)
- bloom1.collect().foreach { each =>
- assert(5 == each.length)
- assert("a1".equals(each.get(0)))
- assert("d1".equals(new String(each.getAs[Array[Byte]](3))))
- if ("b1".equals(each.get(1))) {
- assert("c1".equals(each.get(2)))
- assert("e1".equals(new String(each.getAs[Array[Byte]](4))))
- } else {
- assert(false)
- }
- }
-
- val result = sql(s"show datamap on table $tableName").cache()
-
- checkAnswer(sql(s"show datamap on table $tableName"),
- Seq(Row(datamapName, "bloomfilter", s"default.$tableName", "'bloom_fpp'='0.001', 'bloom_size'='32000', 'index_columns'='d'", "ENABLED", "NA"),
- Row(datamapName2, "bloomfilter", s"default.$tableName", "'index_columns'='e'", "ENABLED", "NA")))
- result.unpersist()
- sql(s"drop table if exists $tableName")
- }
+
test("test don't support timeseries on binary data type") {
val tableName = "datamapshowtest"
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
index bf4106b..d4524d0 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/datamap/CarbonCreateDataMapCommand.scala
@@ -27,6 +27,7 @@ import org.apache.carbondata.common.logging.LogServiceFactory
import org.apache.carbondata.core.datamap.{DataMapProvider, DataMapStoreManager}
import org.apache.carbondata.core.datamap.status.DataMapStatusManager
import org.apache.carbondata.core.metadata.ColumnarFormatVersion
+import org.apache.carbondata.core.metadata.datatype.DataTypes
import org.apache.carbondata.core.metadata.schema.datamap.{DataMapClassProvider, DataMapProperty}
import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, DataMapSchema}
import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
@@ -145,6 +146,12 @@ case class CarbonCreateDataMapCommand(
"column '%s' already has %s index datamap created",
column.getColName, thisDmProviderName))
} else if (isBloomFilter) {
+ if (column.getDataType == DataTypes.BINARY) {
+ throw new MalformedDataMapCommandException(
+ s"BloomFilter datamap does not support Binary datatype column: ${
+ column.getColName
+ }")
+ }
// if datamap provider is bloomfilter,the index column datatype cannot be complex type
if (column.isComplex) {
throw new MalformedDataMapCommandException(
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
index 06e81bd..bedcc4a 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
@@ -590,6 +590,25 @@ class BloomCoarseGrainDataMapSuite extends QueryTest with BeforeAndAfterAll with
"BloomFilter datamap does not support complex datatype column"))
}
+ test("test create bloomfilter datamap which index column datatype is Binary ") {
+ sql("drop table if exists binaryTable")
+ sql(
+ "CREATE TABLE binaryTable (CUST_ID binary,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB " +
+ "timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 " +
+ "decimal(30,10), DECIMAL_COLUMN2 decimal(36,36),Double_COLUMN1 double, Double_COLUMN2 " +
+ "double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'")
+ val exception: MalformedDataMapCommandException = intercept[MalformedDataMapCommandException] {
+ sql(
+ s"""
+ | CREATE DATAMAP binaryBloom ON TABLE binaryTable
+ | USING 'bloomfilter'
+ | DMProperties('INDEX_COLUMNS'='cust_id', 'BLOOM_SIZE'='640000')
+ | """.stripMargin)
+ }
+ assert(exception.getMessage.equalsIgnoreCase(
+ "BloomFilter datamap does not support binary datatype column: cust_id" ))
+ }
+
test("test create bloom datamap on newly added column") {
val datamap1 = "datamap1"
val datamap2 = "datamap2"