You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by qi...@apache.org on 2018/09/11 09:48:43 UTC
carbondata git commit: [CARBONDATA-2811][BloomDataMap] Add query test
case using search mode on table with bloom filter
Repository: carbondata
Updated Branches:
refs/heads/master 73a5885a4 -> 7c827c0a9
[CARBONDATA-2811][BloomDataMap] Add query test case using search mode on table with bloom filter
This closes #2598
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/7c827c0a
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/7c827c0a
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/7c827c0a
Branch: refs/heads/master
Commit: 7c827c0a96c4f9dbbcbd46f2d5c85833e203b2ca
Parents: 73a5885
Author: Manhua <ke...@qq.com>
Authored: Thu Aug 2 10:21:16 2018 +0800
Committer: QiangCai <qi...@qq.com>
Committed: Tue Sep 11 17:47:41 2018 +0800
----------------------------------------------------------------------
.../BloomCoarseGrainDataMapFunctionSuite.scala | 2 +
.../bloom/BloomCoarseGrainDataMapSuite.scala | 60 ++++++++++++++++++++
2 files changed, 62 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7c827c0a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala
index 7ea89a9..dd2ecb4 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapFunctionSuite.scala
@@ -830,6 +830,8 @@ class BloomCoarseGrainDataMapFunctionSuite extends QueryTest with BeforeAndAfte
sql("drop table if exists test_rcd").collect()
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.BLOCKLET_SIZE,
CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL)
+ CarbonProperties.getInstance().addProperty(CarbonV3DataFormatConstants.BLOCKLET_SIZE_IN_MB,
+ CarbonV3DataFormatConstants.BLOCKLET_SIZE_IN_MB_DEFAULT_VALUE)
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/7c827c0a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
index cca1b67..7cb841f 100644
--- a/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
+++ b/integration/spark2/src/test/scala/org/apache/carbondata/datamap/bloom/BloomCoarseGrainDataMapSuite.scala
@@ -32,6 +32,7 @@ import org.apache.carbondata.core.datamap.status.DataMapStatusManager
import org.apache.carbondata.core.util.CarbonProperties
class BloomCoarseGrainDataMapSuite extends QueryTest with BeforeAndAfterAll with BeforeAndAfterEach {
+ val carbonSession = sqlContext.sparkSession.asInstanceOf[CarbonSession]
val bigFile = s"$resourcesPath/bloom_datamap_input_big.csv"
val smallFile = s"$resourcesPath/bloom_datamap_input_small.csv"
val normalTable = "carbon_normal"
@@ -251,6 +252,61 @@ class BloomCoarseGrainDataMapSuite extends QueryTest with BeforeAndAfterAll with
sql(s"DROP TABLE IF EXISTS $bloomDMSampleTable")
}
+ test("test using search mode to query tabel with bloom datamap") {
+ sql(
+ s"""
+ | CREATE TABLE $normalTable(id INT, name STRING, city STRING, age INT,
+ | s1 STRING, s2 STRING, s3 STRING, s4 STRING, s5 STRING, s6 STRING, s7 STRING, s8 STRING)
+ | STORED BY 'carbondata' TBLPROPERTIES('table_blocksize'='128')
+ | """.stripMargin)
+ sql(
+ s"""
+ | CREATE TABLE $bloomDMSampleTable(id INT, name STRING, city STRING, age INT,
+ | s1 STRING, s2 STRING, s3 STRING, s4 STRING, s5 STRING, s6 STRING, s7 STRING, s8 STRING)
+ | STORED BY 'carbondata' TBLPROPERTIES('table_blocksize'='128')
+ | """.stripMargin)
+
+ // load two segments
+ (1 to 2).foreach { i =>
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$bigFile' INTO TABLE $normalTable
+ | OPTIONS('header'='false')
+ """.stripMargin)
+ sql(
+ s"""
+ | LOAD DATA LOCAL INPATH '$bigFile' INTO TABLE $bloomDMSampleTable
+ | OPTIONS('header'='false')
+ """.stripMargin)
+ }
+
+ sql(
+ s"""
+ | CREATE DATAMAP $dataMapName ON TABLE $bloomDMSampleTable
+ | USING 'bloomfilter'
+ | DMProperties('INDEX_COLUMNS'='city,id', 'BLOOM_SIZE'='640000')
+ """.stripMargin)
+ checkExistence(sql(s"SHOW DATAMAP ON TABLE $bloomDMSampleTable"), true, dataMapName)
+
+ // get answer before search mode is enable
+ val expectedAnswer1 = sql(s"select * from $normalTable where id = 1").collect()
+ val expectedAnswer2 = sql(s"select * from $normalTable where city in ('city_999')").collect()
+
+ carbonSession.startSearchMode()
+ assert(carbonSession.isSearchModeEnabled)
+
+ checkAnswer(
+ sql(s"select * from $bloomDMSampleTable where id = 1"), expectedAnswer1)
+ checkAnswer(
+ sql(s"select * from $bloomDMSampleTable where city in ('city_999')"), expectedAnswer2)
+
+ carbonSession.stopSearchMode()
+ assert(!carbonSession.isSearchModeEnabled)
+
+ sql(s"DROP TABLE IF EXISTS $normalTable")
+ sql(s"DROP TABLE IF EXISTS $bloomDMSampleTable")
+ }
+
ignore("test create bloom datamap with DEFERRED REBUILD, query hit datamap") {
sql(
s"""
@@ -848,6 +904,10 @@ class BloomCoarseGrainDataMapSuite extends QueryTest with BeforeAndAfterAll with
}
override protected def afterAll(): Unit = {
+ // in case of search mode test case failed, stop search mode again
+ if (carbonSession.isSearchModeEnabled) {
+ carbonSession.stopSearchMode()
+ }
deleteFile(bigFile)
deleteFile(smallFile)
sql(s"DROP TABLE IF EXISTS $normalTable")