You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by gv...@apache.org on 2018/06/08 11:40:52 UTC
[27/50] [abbrv] carbondata git commit: [CARBONDATA-2499][Test]
Validate the visible/invisible status of datamap
[CARBONDATA-2499][Test] Validate the visible/invisible status of datamap
This closes #2325
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/1b6ce8cd
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/1b6ce8cd
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/1b6ce8cd
Branch: refs/heads/spark-2.3
Commit: 1b6ce8cdcd95c7502098b46608c51ab1d0cb3689
Parents: ddf3e85
Author: xubo245 <xu...@huawei.com>
Authored: Mon May 21 15:47:24 2018 +0800
Committer: QiangCai <qi...@qq.com>
Committed: Mon May 28 21:23:50 2018 +0800
----------------------------------------------------------------------
.../testsuite/datamap/CGDataMapTestCase.scala | 71 ++++++++++++++----
.../testsuite/datamap/FGDataMapTestCase.scala | 78 +++++++++++++++++++-
.../DataLoadFailAllTypeSortTest.scala | 4 -
3 files changed, 130 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1b6ce8cd/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
index 848acde..b5c3df1 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/CGDataMapTestCase.scala
@@ -22,7 +22,6 @@ import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer
import com.sun.xml.internal.messaging.saaj.util.ByteOutputStream
-import org.apache.hadoop.fs.Path
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
@@ -401,30 +400,70 @@ class CGDataMapTestCase extends QueryTest with BeforeAndAfterAll {
sql(s"DROP TABLE IF EXISTS $tableName")
sql(
s"""
- | CREATE TABLE $tableName(id INT, name STRING, city STRING, age INT)
- | STORED BY 'org.apache.carbondata.format'
- | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ | CREATE TABLE $tableName(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
""".stripMargin)
// register datamap writer
- sql(s"create datamap $dataMapName1 on table $tableName using '${classOf[CGDataMapFactory].getName}' DMPROPERTIES('index_columns'='name')")
- sql(s"create datamap $dataMapName2 on table $tableName using '${classOf[CGDataMapFactory].getName}' DMPROPERTIES('index_columns'='city')")
+ sql(
+ s"""
+ | CREATE DATAMAP $dataMapName1
+ | ON TABLE $tableName
+ | USING '${classOf[CGDataMapFactory].getName}'
+ | DMPROPERTIES('index_columns'='name')
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP $dataMapName2
+ | ON TABLE $tableName
+ | USING '${classOf[CGDataMapFactory].getName}'
+ | DMPROPERTIES('index_columns'='city')
+ """.stripMargin)
sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE $tableName OPTIONS('header'='false')")
+ val df1 = sql(s"EXPLAIN EXTENDED SELECT * FROM $tableName WHERE name='n502670' AND city='c2670'").collect()
+ assert(df1(0).getString(0).contains("CG DataMap"))
+ assert(df1(0).getString(0).contains(dataMapName1))
+ val e11 = intercept[Exception] {
+ assert(df1(0).getString(0).contains(dataMapName2))
+ }
+ assert(e11.getMessage.contains("did not contain \"" + dataMapName2))
// make datamap1 invisible
- sql(s"set ${CarbonCommonConstants.CARBON_DATAMAP_VISIBLE}default.$tableName.$dataMapName1 = false")
- checkAnswer(sql(s"select * from $tableName where name='n502670' and city='c2670'"),
- sql("select * from normal_test where name='n502670' and city='c2670'"))
+ sql(s"SET ${CarbonCommonConstants.CARBON_DATAMAP_VISIBLE}default.$tableName.$dataMapName1 = false")
+ val df2 = sql(s"EXPLAIN EXTENDED SELECT * FROM $tableName WHERE name='n502670' AND city='c2670'").collect()
+ val e = intercept[Exception] {
+ assert(df2(0).getString(0).contains(dataMapName1))
+ }
+ assert(e.getMessage.contains("did not contain \"" + dataMapName1))
+ assert(df2(0).getString(0).contains(dataMapName2))
+ checkAnswer(sql(s"SELECT * FROM $tableName WHERE name='n502670' AND city='c2670'"),
+ sql("SELECT * FROM normal_test WHERE name='n502670' AND city='c2670'"))
// also make datamap2 invisible
- sql(s"set ${CarbonCommonConstants.CARBON_DATAMAP_VISIBLE}default.$tableName.$dataMapName2 = false")
- checkAnswer(sql(s"select * from $tableName where name='n502670' and city='c2670'"),
- sql("select * from normal_test where name='n502670' and city='c2670'"))
+ sql(s"SET ${CarbonCommonConstants.CARBON_DATAMAP_VISIBLE}default.$tableName.$dataMapName2 = false")
+ checkAnswer(sql(s"SELECT * FROM $tableName WHERE name='n502670' AND city='c2670'"),
+ sql("SELECT * FROM normal_test WHERE name='n502670' AND city='c2670'"))
+ val df3 = sql(s"EXPLAIN EXTENDED SELECT * FROM $tableName WHERE name='n502670' AND city='c2670'").collect()
+ val e31 = intercept[Exception] {
+ assert(df3(0).getString(0).contains(dataMapName1))
+ }
+ assert(e31.getMessage.contains("did not contain \"" + dataMapName1))
+ val e32 = intercept[Exception] {
+ assert(df3(0).getString(0).contains(dataMapName2))
+ }
+ assert(e32.getMessage.contains("did not contain \"" + dataMapName2))
// make datamap1,datamap2 visible
- sql(s"set ${CarbonCommonConstants.CARBON_DATAMAP_VISIBLE}default.$tableName.$dataMapName1 = true")
- sql(s"set ${CarbonCommonConstants.CARBON_DATAMAP_VISIBLE}default.$tableName.$dataMapName1 = true")
- checkAnswer(sql(s"select * from $tableName where name='n502670' and city='c2670'"),
- sql("select * from normal_test where name='n502670' and city='c2670'"))
+ sql(s"SET ${CarbonCommonConstants.CARBON_DATAMAP_VISIBLE}default.$tableName.$dataMapName1 = true")
+ sql(s"SET ${CarbonCommonConstants.CARBON_DATAMAP_VISIBLE}default.$tableName.$dataMapName1 = true")
+ checkAnswer(sql(s"SELECT * FROM $tableName WHERE name='n502670' AND city='c2670'"),
+ sql("SELECT * FROM normal_test WHERE name='n502670' AND city='c2670'"))
+ val df4 = sql(s"EXPLAIN EXTENDED SELECT * FROM $tableName WHERE name='n502670' AND city='c2670'").collect()
+ assert(df4(0).getString(0).contains(dataMapName1))
+ val e41 = intercept[Exception] {
+ assert(df3(0).getString(0).contains(dataMapName2))
+ }
+ assert(e41.getMessage.contains("did not contain \"" + dataMapName2))
}
test("test datamap storage in system folder") {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1b6ce8cd/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
index e2642ff..2d666c3 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/FGDataMapTestCase.scala
@@ -22,12 +22,10 @@ import scala.collection.JavaConverters._
import scala.collection.mutable.ArrayBuffer
import com.sun.xml.internal.messaging.saaj.util.ByteOutputStream
+import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
-import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta}
-import org.apache.carbondata.core.datamap.Segment
-import org.apache.carbondata.core.datamap.dev.{DataMapModel, DataMapBuilder, DataMapWriter}
import org.apache.carbondata.core.datamap.{DataMapDistributable, DataMapMeta, Segment}
import org.apache.carbondata.core.datamap.dev.{DataMapModel, DataMapBuilder, DataMapWriter}
import org.apache.carbondata.core.datamap.dev.fgdatamap.{FineGrainBlocklet, FineGrainDataMap, FineGrainDataMapFactory}
@@ -488,9 +486,83 @@ class FGDataMapTestCase extends QueryTest with BeforeAndAfterAll {
sql("select * from normal_test where name='n502670' and city='c2670'"))
}
+ test("test invisible datamap during query") {
+ val tableName = "datamap_testFG"
+ val dataMapName1 = "datamap1"
+ val dataMapName2 = "datamap2"
+ sql(s"DROP TABLE IF EXISTS $tableName")
+ sql(
+ s"""
+ | CREATE TABLE $tableName(id INT, name STRING, city STRING, age INT)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('SORT_COLUMNS'='city,name', 'SORT_SCOPE'='LOCAL_SORT')
+ """.stripMargin)
+ // register datamap writer
+ sql(
+ s"""
+ | CREATE DATAMAP $dataMapName1
+ | ON TABLE $tableName
+ | USING '${classOf[FGDataMapFactory].getName}'
+ | DMPROPERTIES('index_columns'='name')
+ """.stripMargin)
+ sql(
+ s"""
+ | CREATE DATAMAP $dataMapName2
+ | ON TABLE $tableName
+ | USING '${classOf[FGDataMapFactory].getName}'
+ | DMPROPERTIES('index_columns'='city')
+ """.stripMargin)
+ sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE $tableName OPTIONS('header'='false')")
+ val df1 = sql(s"EXPLAIN EXTENDED SELECT * FROM $tableName WHERE name='n502670' AND city='c2670'").collect()
+ assert(df1(0).getString(0).contains("FG DataMap"))
+ assert(df1(0).getString(0).contains(dataMapName1))
+ val e11 = intercept[Exception] {
+ assert(df1(0).getString(0).contains(dataMapName2))
+ }
+ assert(e11.getMessage.contains("did not contain \"" + dataMapName2))
+
+ // make datamap1 invisible
+ sql(s"SET ${CarbonCommonConstants.CARBON_DATAMAP_VISIBLE}default.$tableName.$dataMapName1 = false")
+ val df2 = sql(s"EXPLAIN EXTENDED SELECT * FROM $tableName WHERE name='n502670' AND city='c2670'").collect()
+ val e = intercept[Exception] {
+ assert(df2(0).getString(0).contains(dataMapName1))
+ }
+ assert(e.getMessage.contains("did not contain \"" + dataMapName1))
+ assert(df2(0).getString(0).contains(dataMapName2))
+ checkAnswer(sql(s"SELECT * FROM $tableName WHERE name='n502670' AND city='c2670'"),
+ sql("SELECT * FROM normal_test WHERE name='n502670' AND city='c2670'"))
+
+ // also make datamap2 invisible
+ sql(s"SET ${CarbonCommonConstants.CARBON_DATAMAP_VISIBLE}default.$tableName.$dataMapName2 = false")
+ checkAnswer(sql(s"SELECT * FROM $tableName WHERE name='n502670' AND city='c2670'"),
+ sql("SELECT * FROM normal_test WHERE name='n502670' AND city='c2670'"))
+ val df3 = sql(s"EXPLAIN EXTENDED SELECT * FROM $tableName WHERE name='n502670' AND city='c2670'").collect()
+ val e31 = intercept[Exception] {
+ assert(df3(0).getString(0).contains(dataMapName1))
+ }
+ assert(e31.getMessage.contains("did not contain \"" + dataMapName1))
+ val e32 = intercept[Exception] {
+ assert(df3(0).getString(0).contains(dataMapName2))
+ }
+ assert(e32.getMessage.contains("did not contain \"" + dataMapName2))
+
+ // make datamap1,datamap2 visible
+ sql(s"SET ${CarbonCommonConstants.CARBON_DATAMAP_VISIBLE}default.$tableName.$dataMapName1 = true")
+ sql(s"SET ${CarbonCommonConstants.CARBON_DATAMAP_VISIBLE}default.$tableName.$dataMapName1 = true")
+ checkAnswer(sql(s"SELECT * FROM $tableName WHERE name='n502670' AND city='c2670'"),
+ sql("SELECT * FROM normal_test WHERE name='n502670' AND city='c2670'"))
+ val df4 = sql(s"EXPLAIN EXTENDED SELECT * FROM $tableName WHERE name='n502670' AND city='c2670'").collect()
+ assert(df4(0).getString(0).contains(dataMapName1))
+ val e41 = intercept[Exception] {
+ assert(df3(0).getString(0).contains(dataMapName2))
+ }
+ assert(e41.getMessage.contains("did not contain \"" + dataMapName2))
+ }
+
override protected def afterAll(): Unit = {
CompactionSupportGlobalSortBigFileTest.deleteFile(file2)
sql("DROP TABLE IF EXISTS normal_test")
sql("DROP TABLE IF EXISTS datamap_test")
+ sql("DROP TABLE IF EXISTS datamap_testFG")
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/1b6ce8cd/integration/spark2/src/test/scala/org/apache/spark/carbondata/DataLoadFailAllTypeSortTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/DataLoadFailAllTypeSortTest.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/DataLoadFailAllTypeSortTest.scala
index 121150c..a7dceb4 100644
--- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/DataLoadFailAllTypeSortTest.scala
+++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/DataLoadFailAllTypeSortTest.scala
@@ -17,8 +17,6 @@
package org.apache.spark.carbondata
-import java.io.File
-
import org.apache.spark.sql.common.util.Spark2QueryTest
import org.apache.spark.sql.hive.HiveContext
import org.scalatest.BeforeAndAfterAll
@@ -28,8 +26,6 @@ import org.apache.carbondata.core.util.CarbonProperties
/**
* Test Class for detailed query on timestamp datatypes
- *
- *
*/
class DataLoadFailAllTypeSortTest extends Spark2QueryTest with BeforeAndAfterAll {
var hiveContext: HiveContext = _