You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/08/07 13:09:39 UTC
[15/50] [abbrv] carbondata git commit: [CARBONDATA-2585]disable local
dictionary by default
[CARBONDATA-2585]disable local dictionary by default
make local dictionary false by default
This closes #2570
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/34ca0214
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/34ca0214
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/34ca0214
Branch: refs/heads/external-format
Commit: 34ca021423cff8aa2dd03f98f99ea2c46a5aa9a6
Parents: 9a3b0b1
Author: akashrn5 <ak...@gmail.com>
Authored: Fri Jul 27 12:31:48 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Tue Jul 31 21:35:26 2018 +0530
----------------------------------------------------------------------
.../core/constants/CarbonCommonConstants.java | 2 +-
docs/data-management-on-carbondata.md | 2 +-
...CreateTableWithLocalDictionaryTestCase.scala | 70 ++++++++------------
.../TestNonTransactionalCarbonTable.scala | 43 ++----------
...ransactionalCarbonTableWithComplexType.scala | 10 +--
.../describeTable/TestDescribeTable.scala | 4 +-
.../LocalDictionarySupportAlterTableTest.scala | 52 +++++++--------
.../LocalDictionarySupportCreateTableTest.scala | 58 ++++++++--------
.../LocalDictionarySupportLoadTableTest.scala | 14 ++--
.../table/CarbonDescribeFormattedCommand.scala | 3 +-
10 files changed, 98 insertions(+), 160 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/34ca0214/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 8bf22c9..6d7215e 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -929,7 +929,7 @@ public final class CarbonCommonConstants {
/**
* default value for local dictionary generation
*/
- public static final String LOCAL_DICTIONARY_ENABLE_DEFAULT = "true";
+ public static final String LOCAL_DICTIONARY_ENABLE_DEFAULT = "false";
/**
* Threshold value for local dictionary
http://git-wip-us.apache.org/repos/asf/carbondata/blob/34ca0214/docs/data-management-on-carbondata.md
----------------------------------------------------------------------
diff --git a/docs/data-management-on-carbondata.md b/docs/data-management-on-carbondata.md
index 28bc7d3..6aaaaa3 100644
--- a/docs/data-management-on-carbondata.md
+++ b/docs/data-management-on-carbondata.md
@@ -137,7 +137,7 @@ This tutorial is going to introduce all commands and data operations on CarbonDa
| Properties | Default value | Description |
| ---------- | ------------- | ----------- |
- | LOCAL_DICTIONARY_ENABLE | true | By default, local dictionary will be enabled for the table |
+ | LOCAL_DICTIONARY_ENABLE | false | By default, local dictionary will not be enabled for the table |
| LOCAL_DICTIONARY_THRESHOLD | 10000 | The maximum cardinality for local dictionary generation (range- 1000 to 100000) |
| LOCAL_DICTIONARY_INCLUDE | all no-dictionary string/varchar columns | Columns for which Local Dictionary is generated. |
| LOCAL_DICTIONARY_EXCLUDE | none | Columns for which Local Dictionary is not generated |
http://git-wip-us.apache.org/repos/asf/carbondata/blob/34ca0214/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala
index b6ecef9..43d5956 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala
@@ -40,10 +40,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
val descLoc = sql("describe formatted local1").collect
descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
- case Some(row) => assert(row.get(1).toString.contains("true"))
- }
- descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
- case Some(row) => assert(row.get(1).toString.contains("10000"))
+ case Some(row) => assert(row.get(1).toString.contains("false"))
}
}
@@ -53,7 +50,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_include'='name')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name')
""".
stripMargin)
val descFormatted1 = sql("describe formatted local1").collect
@@ -65,21 +62,6 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
}
}
- test(
- "test local dictionary custom configurations for local dict columns _002")
- {
- sql("drop table if exists local1")
-
- intercept[MalformedCarbonCommandException] {
- sql(
- """
- | CREATE TABLE local1(id int, name string, city string, age int)
- | STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_include'='name,name')
- """.stripMargin)
- }
- }
-
test("test local dictionary custom configurations for local dict columns _003") {
sql("drop table if exists local1")
val exception = intercept[MalformedCarbonCommandException] {
@@ -87,7 +69,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_include'='')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='')
""".
stripMargin)
}
@@ -104,7 +86,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_include'='abc')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='abc')
""".
stripMargin)
}
@@ -121,7 +103,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_include'='id')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='id')
""".
stripMargin)
}
@@ -139,7 +121,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('dictionary_include'='name','local_dictionary_include'='name')
+ | tblproperties('local_dictionary_enable'='true','dictionary_include'='name','local_dictionary_include'='name')
""".
stripMargin)
}
@@ -151,7 +133,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='20000')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000')
""".stripMargin)
val descLoc = sql("describe formatted local1").collect
@@ -170,7 +152,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='-100')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='-100')
""".stripMargin)
val descLoc = sql("describe formatted local1").collect
@@ -186,7 +168,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='21474874811')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='21474874811')
""".stripMargin)
val descLoc = sql("describe formatted local1").collect
@@ -202,7 +184,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='')
""".stripMargin)
val descLoc = sql("describe formatted local1").collect
@@ -218,7 +200,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='hello')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='hello')
""".stripMargin)
val descLoc = sql("describe formatted local1").collect
@@ -235,7 +217,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='20000','local_dictionary_include'='name')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000','local_dictionary_include'='name')
""".stripMargin)
val descLoc = sql("describe formatted local1").collect
@@ -258,7 +240,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='-100','local_dictionary_include'='name')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='-100','local_dictionary_include'='name')
""".stripMargin)
val descLoc = sql("describe formatted local1").collect
@@ -281,7 +263,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='','local_dictionary_include'='name')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='','local_dictionary_include'='name')
""".stripMargin)
val descLoc = sql("describe formatted local1").collect
@@ -304,7 +286,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='vdslv','local_dictionary_include'='name')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='vdslv','local_dictionary_include'='name')
""".stripMargin)
val descLoc = sql("describe formatted local1").collect
@@ -328,7 +310,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='20000','local_dictionary_include'='name,name')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000','local_dictionary_include'='name,name')
""".stripMargin)
}
}
@@ -342,7 +324,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='20000','local_dictionary_include'=' ')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000','local_dictionary_include'=' ')
""".stripMargin)
}
}
@@ -356,7 +338,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='20000','local_dictionary_include'='hello')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000','local_dictionary_include'='hello')
""".stripMargin)
}
}
@@ -370,7 +352,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='20000','local_dictionary_include'='name',
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000','local_dictionary_include'='name',
| 'dictionary_include'='name')
""".stripMargin)
}
@@ -385,7 +367,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='','local_dictionary_include'='name,name')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='','local_dictionary_include'='name,name')
""".stripMargin)
}
}
@@ -399,7 +381,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='-100','local_dictionary_include'='Hello')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='-100','local_dictionary_include'='Hello')
""".stripMargin)
}
}
@@ -413,7 +395,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='23213497321591234324',
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='23213497321591234324',
| 'local_dictionary_include'='name','dictionary_include'='name')
""".stripMargin)
}
@@ -1496,7 +1478,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
val descLoc = sql("describe formatted local1").collect
descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
- case Some(row) => assert(row.get(1).toString.contains("true"))
+ case Some(row) => assert(row.get(1).toString.contains("false"))
}
descLoc.find(_.get(0).toString.contains("SORT_SCOPE")) match {
case Some(row) => assert(row.get(1).toString.contains("global_sort"))
@@ -1516,7 +1498,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
val descLoc = sql("describe formatted local1").collect
descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
- case Some(row) => assert(row.get(1).toString.contains("true"))
+ case Some(row) => assert(row.get(1).toString.contains("false"))
}
descLoc.find(_.get(0).toString.contains("SORT_SCOPE")) match {
case Some(row) => assert(row.get(1).toString.contains("batch_sort"))
@@ -1535,7 +1517,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
val descLoc = sql("describe formatted local1").collect
descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
- case Some(row) => assert(row.get(1).toString.contains("true"))
+ case Some(row) => assert(row.get(1).toString.contains("false"))
}
descLoc.find(_.get(0).toString.contains("SORT_SCOPE")) match {
case Some(row) => assert(row.get(1).toString.contains("no_sort"))
@@ -1554,7 +1536,7 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
val descLoc = sql("describe formatted local1").collect
descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
- case Some(row) => assert(row.get(1).toString.contains("true"))
+ case Some(row) => assert(row.get(1).toString.contains("false"))
}
descLoc.find(_.get(0).toString.contains("SORT_SCOPE")) match {
case Some(row) => assert(row.get(1).toString.contains("local_sort"))
http://git-wip-us.apache.org/repos/asf/carbondata/blob/34ca0214/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
index 62c3df6..8a1d465 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTable.scala
@@ -2328,15 +2328,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
|'$writerPath' """.stripMargin)
val descLoc = sql("describe formatted sdkTable").collect
descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
- case Some(row) => assert(row.get(1).toString.contains("true"))
- case None => assert(false)
- }
- descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
- case Some(row) => assert(row.get(1).toString.contains("10000"))
- case None => assert(false)
- }
- descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
- case Some(row) => assert(row.get(1).toString.contains("name,surname"))
+ case Some(row) => assert(row.get(1).toString.contains("false"))
case None => assert(false)
}
FileUtils.deleteDirectory(new File(writerPath))
@@ -2357,15 +2349,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
|'$writerPath' """.stripMargin)
val descLoc = sql("describe formatted sdkTable").collect
descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
- case Some(row) => assert(row.get(1).toString.contains("true"))
- case None => assert(false)
- }
- descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
- case Some(row) => assert(row.get(1).toString.contains("10000"))
- case None => assert(false)
- }
- descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
- case Some(row) => assert(row.get(1).toString.contains("name,surname"))
+ case Some(row) => assert(row.get(1).toString.contains("false"))
case None => assert(false)
}
FileUtils.deleteDirectory(new File(writerPath))
@@ -2386,15 +2370,7 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
|'$writerPath' """.stripMargin)
val descLoc = sql("describe formatted sdkTable").collect
descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
- case Some(row) => assert(row.get(1).toString.contains("true"))
- case None => assert(false)
- }
- descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
- case Some(row) => assert(row.get(1).toString.contains("10000"))
- case None => assert(false)
- }
- descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
- case Some(row) => assert(row.get(1).toString.contains("name,surname"))
+ case Some(row) => assert(row.get(1).toString.contains("false"))
case None => assert(false)
}
FileUtils.deleteDirectory(new File(writerPath))
@@ -2416,21 +2392,12 @@ class TestNonTransactionalCarbonTable extends QueryTest with BeforeAndAfterAll {
FileUtils.deleteDirectory(new File(writerPath))
sql("insert into sdkTable select 's1','s2',23 ")
assert(FileFactory.getCarbonFile(writerPath).exists())
- assert(testUtil.checkForLocalDictionary(testUtil.getDimRawChunk(0,writerPath)))
+ assert(!testUtil.checkForLocalDictionary(testUtil.getDimRawChunk(0,writerPath)))
val descLoc = sql("describe formatted sdkTable").collect
descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
- case Some(row) => assert(row.get(1).toString.contains("true"))
+ case Some(row) => assert(row.get(1).toString.contains("false"))
case None => assert(false)
}
- descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
- case Some(row) => assert(row.get(1).toString.contains("10000"))
- case None => assert(false)
- }
- descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
- case Some(row) => assert(row.get(1).toString.contains("name,surname"))
- case None => assert(false)
- }
-
checkAnswer(sql("select count(*) from sdkTable"), Seq(Row(1)))
FileUtils.deleteDirectory(new File(writerPath))
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/34ca0214/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableWithComplexType.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableWithComplexType.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableWithComplexType.scala
index 7593f38..62ba03e 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableWithComplexType.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestNonTransactionalCarbonTableWithComplexType.scala
@@ -218,15 +218,7 @@ class TestNonTransactionalCarbonTableWithComplexType extends QueryTest with Befo
sql("describe formatted localComplex").show(30, false)
val descLoc = sql("describe formatted localComplex").collect
descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
- case Some(row) => assert(row.get(1).toString.contains("true"))
- case None => assert(false)
- }
- descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
- case Some(row) => assert(row.get(1).toString.contains("10000"))
- case None => assert(false)
- }
- descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
- case Some(row) => assert(row.get(1).toString.contains("name,val1.val2.street,val1.val2.city,val1.val2.WindSpeed,val1.val2.year"))
+ case Some(row) => assert(row.get(1).toString.contains("false"))
case None => assert(false)
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/34ca0214/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala
index 93f1736..d0d89aa 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/describeTable/TestDescribeTable.scala
@@ -51,10 +51,10 @@ class TestDescribeTable extends QueryTest with BeforeAndAfterAll {
test("test describe formatted table desc1") {
val resultCol = Seq("", "", "##Detailed Column property", "##Detailed Table Information", "ADAPTIVE", "CARBON Store Path", "Comment", "Database Name", "Last Update Time",
- "SORT_COLUMNS", "SORT_SCOPE", "CACHE_LEVEL", "Streaming", "Table Block Size", "Local Dictionary Enabled", "Local Dictionary Include", "Local Dictionary Threshold","Table Data Size", "Table Index Size", "Table Name", "dec2col1", "dec2col2", "dec2col3", "dec2col4")
+ "SORT_COLUMNS", "SORT_SCOPE", "CACHE_LEVEL", "Streaming", "Table Block Size", "Local Dictionary Enabled","Table Data Size", "Table Index Size", "Table Name", "dec2col1", "dec2col2", "dec2col3", "dec2col4")
val resultRow: Seq[Row] = resultCol map(propName => Row(f"$propName%-36s"))
checkAnswer(sql("desc formatted DESC1").select("col_name"), resultRow)
- assert(sql("desc formatted desc1").count() == 24)
+ assert(sql("desc formatted desc1").count() == 22)
}
test("test describe formatted for partition table") {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/34ca0214/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportAlterTableTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportAlterTableTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportAlterTableTest.scala
index 47a58ae..373b309 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportAlterTableTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportAlterTableTest.scala
@@ -232,7 +232,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
sql(
"""
| CREATE TABLE local1(id int, name string, city string, age int)
- | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_threshold'='300000')
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='300000')
""".stripMargin)
val descLoc = sql("describe formatted local1").collect
descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
@@ -243,7 +243,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
sql(
"""
| CREATE TABLE local1(id int, name string, city string, age int)
- | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_threshold'='500')
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='500')
""".stripMargin)
val descLoc1 = sql("describe formatted local1").collect
descLoc1.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
@@ -278,12 +278,12 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
}
}
- test("test local dictionary foer varchar datatype columns") {
+ test("test local dictionary for varchar datatype columns") {
sql("drop table if exists local1")
sql(
"""
| CREATE TABLE local1(id int, name string, city string, age int)
- | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_include'='city',
+ | STORED BY 'org.apache.carbondata.format' tblproperties('local_dictionary_enable'='true','local_dictionary_include'='city',
| 'LONG_STRING_COLUMNS'='city')
""".stripMargin)
val descLoc = sql("describe formatted local1").collect
@@ -302,7 +302,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
sql(
"""
| CREATE TABLE local1(id int, name string, city string, age int)
- | STORED BY 'carbondata'
+ | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true')
""".stripMargin)
val descLoc = sql("describe formatted local1").collect
@@ -325,7 +325,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
sql(
"""
| CREATE TABLE local1(id int, name string, city string, age int)
- | STORED BY 'carbondata' tblproperties('local_dictionary_threshold'='300000')
+ | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='300000')
""".stripMargin)
val descLoc = sql("describe formatted local1").collect
@@ -340,7 +340,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
sql(
"""
| CREATE TABLE local1(id int, name string, city string, age int)
- | STORED BY 'carbondata' tblproperties('local_dictionary_threshold'='300000')
+ | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='300000')
""".stripMargin)
val descLoc1 = sql("describe formatted local1").collect
@@ -367,7 +367,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
sql(
"""
| CREATE TABLE local1(id int, name string, city string, age int)
- | STORED BY 'carbondata'
+ | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true')
""".stripMargin)
val descLoc1 = sql("describe formatted local1").collect
@@ -393,7 +393,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
sql(
"""
| CREATE TABLE local1(id int, name string, city string, age int)
- | STORED BY 'carbondata'
+ | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true')
""".stripMargin)
val descLoc1 = sql("describe formatted local1").collect
@@ -418,7 +418,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
sql(
"""
| CREATE TABLE local1(id int, name string, city string, age int)
- | STORED BY 'carbondata'
+ | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true')
""".stripMargin)
val descLoc1 = sql("describe formatted local1").collect
@@ -447,7 +447,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
sql(
"""
| CREATE TABLE local1(id int, name string, city string, age int)
- | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name')
+ | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name')
""".stripMargin)
val descLoc1 = sql("describe formatted local1").collect
@@ -473,7 +473,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
sql(
"""
| CREATE TABLE local1(id int, name string, city string, age int)
- | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name')
+ | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name')
""".stripMargin)
val descLoc1 = sql("describe formatted local1").collect
@@ -499,7 +499,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
sql(
"""
| CREATE TABLE local1(id int, name string, city string, age int)
- | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name')
+ | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name')
""".stripMargin)
val descLoc1 = sql("describe formatted local1").collect
@@ -529,7 +529,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
sql(
"""
| CREATE TABLE local1(id int, name string, city string, age int)
- | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name')
+ | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name')
""".stripMargin)
val descLoc1 = sql("describe formatted local1").collect
@@ -551,7 +551,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
sql(
"""
| CREATE TABLE local1(id int, name string, city string, age int)
- | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name')
+ | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name')
""".stripMargin)
val descLoc1 = sql("describe formatted local1").collect
@@ -570,7 +570,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
sql(
"""
| CREATE TABLE local1(id int, name string, city string, age int)
- | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name')
+ | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name')
""".stripMargin)
val descLoc1 = sql("describe formatted local1").collect
@@ -589,7 +589,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
sql(
"""
| CREATE TABLE local1(id int, name string, city string, age int)
- | STORED BY 'carbondata' tblproperties('dictionary_include'='name')
+ | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','dictionary_include'='name')
""".stripMargin)
val descLoc1 = sql("describe formatted local1").collect
@@ -608,7 +608,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
sql(
"""
| CREATE TABLE local1(id int, name string, city string, age int)
- | STORED BY 'carbondata' tblproperties('local_dictionary_exclude'='name')
+ | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_exclude'='name')
""".stripMargin)
val descLoc1 = sql("describe formatted local1").collect
@@ -637,7 +637,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
sql(
"""
| CREATE TABLE local1(id int, name string, city string, age int)
- | STORED BY 'carbondata' tblproperties('local_dictionary_exclude'='name')
+ | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_exclude'='name')
""".stripMargin)
val descLoc1 = sql("describe formatted local1").collect
@@ -656,7 +656,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
sql(
"""
| CREATE TABLE local1(id int, name string, city string, age int)
- | STORED BY 'carbondata' tblproperties('dictionary_include'='name')
+ | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','dictionary_include'='name')
""".stripMargin)
val descLoc1 = sql("describe formatted local1").collect
@@ -675,7 +675,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
sql(
"""
| CREATE TABLE local1(id int, name string, city string, age int)
- | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name')
+ | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name')
""".stripMargin)
val descLoc1 = sql("describe formatted local1").collect
@@ -694,7 +694,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
sql(
"""
| CREATE TABLE local1(id int, name string, city string, age int)
- | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name')
+ | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name')
""".stripMargin)
val descLoc1 = sql("describe formatted local1").collect
@@ -745,7 +745,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
sql(
"""
| CREATE TABLE local1(id int, name string, city string, age int)
- | STORED BY 'carbondata' tblproperties('local_dictionary_include'='name','local_dictionary_exclude'='city')
+ | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name','local_dictionary_exclude'='city')
""".stripMargin)
sql("alter table local1 unset tblproperties('local_dictionary_exclude')")
@@ -878,7 +878,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
sql("alter table local1 unset tblproperties('local_dictionary_enable')")
val descLoc2 = sql("describe formatted local1").collect
descLoc2.find(_.get(0).toString.contains("Local Dictionary Enable")) match {
- case Some(row) => assert(row.get(1).toString.contains("true"))
+ case Some(row) => assert(row.get(1).toString.contains("false"))
case None => assert(false)
}
}
@@ -888,7 +888,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
sql(
"""
| CREATE TABLE local1(id int, name string, city string, age int,add string)
- | STORED BY 'carbondata' tblproperties('local_dictionary_include'='city')
+ | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_include'='city')
""".stripMargin)
val descLoc1 = sql("describe formatted local1").collect
@@ -909,7 +909,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
sql(
"""
| CREATE TABLE local1(id int, name string, city string, age int,add string)
- | STORED BY 'carbondata' tblproperties('local_dictionary_include'='city',
+ | STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true','local_dictionary_include'='city',
| 'local_dictionary_exclude'='name')
""".stripMargin)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/34ca0214/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala
index eec1582..52c18d0 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala
@@ -40,11 +40,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
val descLoc = sql("describe formatted local1").collect
descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
- case Some(row) => assert(row.get(1).toString.contains("true"))
- case None => assert(false)
- }
- descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
- case Some(row) => assert(row.get(1).toString.contains("10000"))
+ case Some(row) => assert(row.get(1).toString.contains("false"))
case None => assert(false)
}
}
@@ -55,7 +51,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_include'='name')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name')
""".
stripMargin)
val descFormatted1 = sql("describe formatted local1").collect
@@ -77,7 +73,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_include'='name,name')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='name,name')
""".stripMargin)
}
}
@@ -89,7 +85,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_include'='')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='')
""".
stripMargin)
}
@@ -106,7 +102,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_include'='abc')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='abc')
""".
stripMargin)
}
@@ -123,7 +119,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_include'='id')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_include'='id')
""".
stripMargin)
}
@@ -142,7 +138,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('dictionary_include'='name','local_dictionary_include'='name')
+ | tblproperties('local_dictionary_enable'='true','dictionary_include'='name','local_dictionary_include'='name')
""".
stripMargin)
}
@@ -154,7 +150,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='20000')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000')
""".stripMargin)
val descLoc = sql("describe formatted local1").collect
@@ -174,7 +170,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='-100')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='-100')
""".stripMargin)
val descLoc = sql("describe formatted local1").collect
@@ -190,7 +186,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='21474874811')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='21474874811')
""".stripMargin)
val descLoc = sql("describe formatted local1").collect
@@ -206,7 +202,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='')
""".stripMargin)
val descLoc = sql("describe formatted local1").collect
@@ -222,7 +218,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='hello')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='hello')
""".stripMargin)
val descLoc = sql("describe formatted local1").collect
@@ -239,7 +235,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='20000','local_dictionary_include'='name')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000','local_dictionary_include'='name')
""".stripMargin)
val descLoc = sql("describe formatted local1").collect
@@ -264,7 +260,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='-100','local_dictionary_include'='name')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='-100','local_dictionary_include'='name')
""".stripMargin)
val descLoc = sql("describe formatted local1").collect
@@ -289,7 +285,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='','local_dictionary_include'='name')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='','local_dictionary_include'='name')
""".stripMargin)
val descLoc = sql("describe formatted local1").collect
@@ -314,7 +310,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='vdslv','local_dictionary_include'='name')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='vdslv','local_dictionary_include'='name')
""".stripMargin)
val descLoc = sql("describe formatted local1").collect
@@ -340,7 +336,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='20000','local_dictionary_include'='name,
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000','local_dictionary_include'='name,
| name')
""".stripMargin)
}
@@ -354,7 +350,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='20000','local_dictionary_include'=' ')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000','local_dictionary_include'=' ')
""".stripMargin)
}
}
@@ -367,7 +363,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='20000','local_dictionary_include'='hello')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000','local_dictionary_include'='hello')
""".stripMargin)
}
}
@@ -380,7 +376,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='20000','local_dictionary_include'='name',
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='20000','local_dictionary_include'='name',
| 'dictionary_include'='name')
""".stripMargin)
}
@@ -394,7 +390,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='','local_dictionary_include'='name,name')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='','local_dictionary_include'='name,name')
""".stripMargin)
}
}
@@ -407,7 +403,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='-100','local_dictionary_include'='Hello')
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='-100','local_dictionary_include'='Hello')
""".stripMargin)
}
}
@@ -420,7 +416,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('local_dictionary_threshold'='23213497321591234324',
+ | tblproperties('local_dictionary_enable'='true','local_dictionary_threshold'='23213497321591234324',
| 'local_dictionary_include'='name','dictionary_include'='name')
""".stripMargin)
}
@@ -1504,7 +1500,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"""
| CREATE TABLE local1(id int, name string, city string, age int)
| STORED BY 'org.apache.carbondata.format'
- | tblproperties('dictionary_include'='city','sort_scope'='global_sort',
+ | tblproperties('local_dictionary_enable'='true','dictionary_include'='city','sort_scope'='global_sort',
| 'sort_columns'='city,name')
""".stripMargin)
@@ -1533,7 +1529,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
val descLoc = sql("describe formatted local1").collect
descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
- case Some(row) => assert(row.get(1).toString.contains("true"))
+ case Some(row) => assert(row.get(1).toString.contains("false"))
case None => assert(false)
}
descLoc.find(_.get(0).toString.contains("SORT_SCOPE")) match {
@@ -1554,7 +1550,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
val descLoc = sql("describe formatted local1").collect
descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
- case Some(row) => assert(row.get(1).toString.contains("true"))
+ case Some(row) => assert(row.get(1).toString.contains("false"))
case None => assert(false)
}
descLoc.find(_.get(0).toString.contains("SORT_SCOPE")) match {
@@ -1575,7 +1571,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
val descLoc = sql("describe formatted local1").collect
descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
- case Some(row) => assert(row.get(1).toString.contains("true"))
+ case Some(row) => assert(row.get(1).toString.contains("false"))
case None => assert(false)
}
descLoc.find(_.get(0).toString.contains("SORT_SCOPE")) match {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/34ca0214/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportLoadTableTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportLoadTableTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportLoadTableTest.scala
index a3ab851..59586c0 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportLoadTableTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportLoadTableTest.scala
@@ -65,14 +65,14 @@ class LocalDictionarySupportLoadTableTest extends QueryTest with BeforeAndAfterA
sql("drop table if exists local2")
sql(
"CREATE TABLE local2(name string) STORED BY 'carbondata' tblproperties" +
- "('local_dictionary_threshold'='9001','local_dictionary_include'='name')")
+ "('local_dictionary_enable'='true','local_dictionary_threshold'='9001','local_dictionary_include'='name')")
sql("load data inpath '" + file1 + "' into table local2 OPTIONS('header'='false')")
assert(checkForLocalDictionary(getDimRawChunk(0)))
}
test("test successful local dictionary generation for default configs") {
sql("drop table if exists local2")
- sql("CREATE TABLE local2(name string) STORED BY 'carbondata'")
+ sql("CREATE TABLE local2(name string) STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true')")
sql("load data inpath '" + file1 + "' into table local2 OPTIONS('header'='false')")
assert(checkForLocalDictionary(getDimRawChunk(0)))
}
@@ -90,7 +90,7 @@ class LocalDictionarySupportLoadTableTest extends QueryTest with BeforeAndAfterA
sql("drop table if exists local2")
sql(
"CREATE TABLE local2(name string) STORED BY 'carbondata' tblproperties" +
- "('dictionary_exclude'='name')")
+ "('local_dictionary_enable'='true','dictionary_exclude'='name')")
sql("load data inpath '" + file1 + "' into table local2 OPTIONS('header'='false')")
assert(checkForLocalDictionary(getDimRawChunk(0)))
}
@@ -108,7 +108,7 @@ class LocalDictionarySupportLoadTableTest extends QueryTest with BeforeAndAfterA
sql("drop table if exists local2")
sql(
"CREATE TABLE local2(name string) STORED BY 'carbondata' tblproperties" +
- "('local_dictionary_include'='name','local_dictionary_threshold'='300000')")
+ "('local_dictionary_enable'='true','local_dictionary_include'='name','local_dictionary_threshold'='300000')")
sql("load data inpath '" + file1 + "' into table local2 OPTIONS('header'='false')")
assert(checkForLocalDictionary(getDimRawChunk(0)))
}
@@ -117,7 +117,7 @@ class LocalDictionarySupportLoadTableTest extends QueryTest with BeforeAndAfterA
sql("drop table if exists local2")
sql(
"CREATE TABLE local2(name string, age string) STORED BY 'carbondata' tblproperties" +
- "('local_dictionary_include'='name', 'local_dictionary_exclude'='age')")
+ "('local_dictionary_enable'='true','local_dictionary_include'='name', 'local_dictionary_exclude'='age')")
sql("insert into table local2 values('vishal', '30')")
assert(checkForLocalDictionary(getDimRawChunk(0)))
assert(!checkForLocalDictionary(getDimRawChunk(1)))
@@ -127,7 +127,7 @@ class LocalDictionarySupportLoadTableTest extends QueryTest with BeforeAndAfterA
sql("drop table if exists local2")
sql(
"CREATE TABLE local2(name struct<i:string,s:string>) STORED BY 'carbondata' tblproperties" +
- "('local_dictionary_include'='name')")
+ "('local_dictionary_enable'='true','local_dictionary_include'='name')")
sql("load data inpath '" + file2 +
"' into table local2 OPTIONS('header'='false','COMPLEX_DELIMITER_LEVEL_1'='$', " +
"'COMPLEX_DELIMITER_LEVEL_2'=':')")
@@ -138,7 +138,7 @@ class LocalDictionarySupportLoadTableTest extends QueryTest with BeforeAndAfterA
test("test to validate local dictionary values"){
sql("drop table if exists local2")
- sql("CREATE TABLE local2(name string) STORED BY 'carbondata'")
+ sql("CREATE TABLE local2(name string) STORED BY 'carbondata' tblproperties('local_dictionary_enable'='true')")
sql("load data inpath '" + resourcesPath + "/localdictionary.csv" + "' into table local2")
val dimRawChunk = getDimRawChunk(0)
val dictionaryData = Array("vishal", "kumar", "akash", "praveen", "brijoo")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/34ca0214/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
index 41dfea5..9b9e8bd 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
@@ -160,7 +160,8 @@ private[sql] case class CarbonDescribeFormattedCommand(
}
}
} else {
- results ++= Seq(("Local Dictionary Enabled", "false", ""))
+ results ++=
+ Seq(("Local Dictionary Enabled", CarbonCommonConstants.LOCAL_DICTIONARY_ENABLE_DEFAULT, ""))
}
/**