You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ku...@apache.org on 2018/07/09 06:37:14 UTC
carbondata git commit: [CARBONDATA-2689] Added validations for
complex columns in alter set statements
Repository: carbondata
Updated Branches:
refs/heads/master 7924d5e37 -> f0c77e5f5
[CARBONDATA-2689] Added validations for complex columns in alter set statements
Issue: Alter set statements were not validating complex dataType columns correctly.
Fix: Added a recursive method to validate string and varchar child columns of complex dataType columns.
This closes #2450
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/f0c77e5f
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/f0c77e5f
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/f0c77e5f
Branch: refs/heads/master
Commit: f0c77e5f54706476bafd7c802ca90cb09d277533
Parents: 7924d5e
Author: praveenmeenakshi56 <pr...@gmail.com>
Authored: Fri Jul 6 15:07:19 2018 +0530
Committer: kunal642 <ku...@gmail.com>
Committed: Mon Jul 9 12:05:44 2018 +0530
----------------------------------------------------------------------
.../apache/carbondata/core/util/CarbonUtil.java | 10 +-
...CreateTableWithLocalDictionaryTestCase.scala | 48 +-
.../LocalDictionarySupportAlterTableTest.scala | 56 +-
.../LocalDictionarySupportCreateTableTest.scala | 553 +++++++++----------
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 18 +-
.../command/carbonTableSchemaCommon.scala | 3 +
.../org/apache/spark/util/AlterTableUtil.scala | 68 +--
7 files changed, 379 insertions(+), 377 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f0c77e5f/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 9423e28..eaa2b74 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -3141,8 +3141,9 @@ public final class CarbonUtil {
} else {
continue;
}
- if (column.isDimensionColumn() && (column.getDataType().equals(DataTypes.STRING) || column
- .getDataType().equals(DataTypes.VARCHAR)) && !column.hasEncoding(Encoding.DICTIONARY)
+ if (column.isDimensionColumn() && (column.getDataType().equals(DataTypes.STRING) ||
+ column.getDataType().equals(DataTypes.VARCHAR)) &&
+ !column.hasEncoding(Encoding.DICTIONARY)
&& localDictIncludeColumns.toLowerCase()
.contains(column.getColumnName().toLowerCase())) {
for (String dictColumn : listOfDictionaryIncludeColumns) {
@@ -3172,8 +3173,9 @@ public final class CarbonUtil {
dimensionOrdinal++;
setLocalDictForComplexColumns(allColumns, dimensionOrdinal, column.getNumberOfChild());
} else {
- if (column.isDimensionColumn() && (column.getDataType().equals(DataTypes.STRING) || column
- .getDataType().equals(DataTypes.VARCHAR)) && !column.hasEncoding(Encoding.DICTIONARY)) {
+ if (column.isDimensionColumn() && (column.getDataType().equals(DataTypes.STRING) ||
+ column.getDataType().equals(DataTypes.VARCHAR)) &&
+ !column.hasEncoding(Encoding.DICTIONARY)) {
column.setLocalDictColumn(true);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f0c77e5f/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala
index b787d70..b6ecef9 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/CreateTableWithLocalDictionaryTestCase.scala
@@ -93,8 +93,8 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
}
assert(exception.getMessage
.contains(
- "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does not exist in table. Please " +
- "check the DDL."))
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does not exist in table. Please check " +
+ "the DDL."))
}
test("test local dictionary custom configurations for local dict columns _004") {
@@ -110,8 +110,8 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
}
assert(exception1.getMessage
.contains(
- "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. " +
- "Please check the DDL."))
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. Please check " +
+ "the DDL."))
}
test("test local dictionary custom configurations for local dict columns _005") {
@@ -127,9 +127,9 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
}
assert(exception.getMessage
.contains(
- "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex datatype column. " +
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a string/complex/varchar datatype column. " +
"LOCAL_DICTIONARY_COLUMN should " +
- "be no dictionary string/complex datatype column"))
+ "be no dictionary string/complex/varchar datatype column"))
}
test("test local dictionary custom configurations for local dict columns _006") {
@@ -483,8 +483,8 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
}
assert(exception.getMessage
.contains(
- "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does not exist in table. " +
- "Please check the DDL."))
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: does not exist in table. Please check " +
+ "the DDL."))
}
@@ -501,8 +501,8 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
}
assert(exception1.getMessage
.contains(
- "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. " +
- "Please check the DDL."))
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. Please check " +
+ "the DDL."))
}
test("test local dictionary custom configurations when enabled for local dict columns _005") {
@@ -518,9 +518,9 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
}
assert(exception.getMessage
.contains(
- "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex datatype column. " +
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a string/complex/varchar datatype column. " +
"LOCAL_DICTIONARY_COLUMN should " +
- "be no dictionary string/complex datatype column"))
+ "be no dictionary string/complex/varchar datatype column"))
}
test("test local dictionary custom configurations when enabled for local dict columns _006") {
@@ -600,8 +600,8 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
}
assert(exception1.getMessage
.contains(
- "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. " +
- "Please check the DDL."))
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. Please check " +
+ "the DDL."))
}
test("test local dictionary custom configurations when local_dictionary_exclude is configured _005") {
@@ -617,9 +617,9 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
}
assert(exception.getMessage
.contains(
- "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex datatype column. " +
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a string/complex/varchar datatype column. " +
"LOCAL_DICTIONARY_COLUMN should " +
- "be no dictionary string/complex datatype column"))
+ "be no dictionary string/complex/varchar datatype column"))
}
test("test local dictionary custom configurations when local_dictionary_exclude is configured _006") {
@@ -1674,8 +1674,8 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
}
assert(exception1.getMessage
.contains(
- "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. " +
- "Please check the DDL."))
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. Please check " +
+ "the DDL."))
}
test("test CTAS statements for local dictionary custom configurations when enabled for local dict columns _005") {
@@ -1697,9 +1697,9 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
}
assert(exception.getMessage
.contains(
- "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex datatype column. " +
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a string/complex/varchar datatype column. " +
"LOCAL_DICTIONARY_COLUMN should " +
- "be no dictionary string/complex datatype column"))
+ "be no dictionary string/complex/varchar datatype column"))
}
test("test CTAS statements for local dictionary custom configurations when enabled for local dict columns _006") {
@@ -1806,8 +1806,8 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
}
assert(exception1.getMessage
.contains(
- "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. " +
- "Please check the DDL."))
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc does not exist in table. Please check " +
+ "the DDL."))
}
test("test CTAS statements for local dictionary custom configurations when local_dictionary_exclude is configured _005") {
@@ -1829,9 +1829,9 @@ class CreateTableWithLocalDictionaryTestCase extends QueryTest with BeforeAndAft
}
assert(exception.getMessage
.contains(
- "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex datatype column. " +
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a string/complex/varchar datatype column. " +
"LOCAL_DICTIONARY_COLUMN should " +
- "be no dictionary string/complex datatype column"))
+ "be no dictionary string/complex/varchar datatype column"))
}
test("test CTAS statements for local dictionary custom configurations when local_dictionary_exclude is configured _006") {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f0c77e5f/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportAlterTableTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportAlterTableTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportAlterTableTest.scala
index 341a28a..c7dae39 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportAlterTableTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportAlterTableTest.scala
@@ -150,8 +150,8 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
}
assert(exception.getMessage
.contains(
- "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc is not a String/complex " +
- "datatype column. LOCAL_DICTIONARY_COLUMN should be no dictionary string/complex datatype" +
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: abc is not a string/complex/varchar " +
+ "datatype column. LOCAL_DICTIONARY_COLUMN should be no dictionary string/complex/varchar datatype" +
" column.Please check the DDL."))
}
@@ -537,7 +537,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
val exception1 = intercept[Exception] {
sql("alter table local1 set tblproperties('local_dictionary_exclude'='id')")
}
- assert(exception1.getMessage.contains("LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex datatype column. LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE should be no dictionary string/complex datatype column."))
+ assert(exception1.getMessage.contains("LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a string/complex/varchar datatype column. LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE should be no dictionary string/complex/varchar datatype column."))
}
test("test alter set for local dictionary _010") {
@@ -790,9 +790,8 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
| 'local_dictionary_enable'='true')
""".
stripMargin)
- intercept[Exception] {
- sql("alter table local1 set tblproperties('local_dictionary_exclude'='dcity')")
- }
+ sql("alter table local1 set tblproperties('local_dictionary_exclude'='dcity')")
+
intercept[Exception] {
sql("alter table local1 set tblproperties('local_dictionary_include'='st')")
}
@@ -832,7 +831,7 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
sql("alter table local1 unset tblproperties('local_dictionary_include')")
val descLoc2 = sql("describe formatted local1").collect
descLoc2.find(_.get(0).toString.contains("Local Dictionary Include")) match {
- case Some(row) => assert(row.get(1).toString.contains("name,city"))
+ case Some(row) => assert(row.get(1).toString.contains("name,city,add"))
}
}
@@ -1119,9 +1118,50 @@ class LocalDictionarySupportAlterTableTest extends QueryTest with BeforeAndAfter
}
}
+ test("test alter for local dictionary with custom configurations when varchar columns are specified _001") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string,city string, st array<struct<si:string,sd:int>>)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('long_string_columns'='name','local_dictionary_enable'='true')
+ """.stripMargin)
+ sql("alter table local1 set tblproperties('local_dictionary_exclude'='st,name')")
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("10000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Exclude")) match {
+ case Some(row) => assert(row.get(1).toString.contains("st.val.si,name"))
+ }
+ }
+
+ test("test alter for local dictionary with custom configurations when varchar columns are specified _002") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string,city string, st array<struct<si:string,sd:int>>)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('long_string_columns'='name','local_dictionary_enable'='true',
+ | 'local_dictionary_include'='name')
+ """.stripMargin)
+ sql("alter table local1 set tblproperties('local_dictionary_include'='st,name')")
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("10000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("st.val.si,name"))
+ }
+ }
override protected def afterAll(): Unit = {
sql("DROP TABLE IF EXISTS LOCAL1")
}
-
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f0c77e5f/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala
index 5398613..32fab2c 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/localdictionary/LocalDictionarySupportCreateTableTest.scala
@@ -65,9 +65,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations for local dict columns _002")
- {
+ test("test local dictionary custom configurations for local dict columns _002") {
sql("drop table if exists local1")
intercept[MalformedCarbonCommandException] {
@@ -127,10 +125,10 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
assert(exception.getMessage
.contains(
- "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex " +
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a string/complex/varchar " +
"datatype column. " +
"LOCAL_DICTIONARY_COLUMN should " +
- "be no dictionary string/complex datatype column"))
+ "be no dictionary string/complex/varchar datatype column"))
}
test("test local dictionary custom configurations for local dict columns _006") {
@@ -164,8 +162,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test("test local dictionary custom configurations for local dict threshold _002")
- {
+ test("test local dictionary custom configurations for local dict threshold _002") {
sql("drop table if exists local1")
sql(
"""
@@ -180,8 +177,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test("test local dictionary custom configurations for local dict threshold _003")
- {
+ test("test local dictionary custom configurations for local dict threshold _003") {
sql("drop table if exists local1")
sql(
"""
@@ -196,8 +192,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test("test local dictionary custom configurations for local dict threshold _004")
- {
+ test("test local dictionary custom configurations for local dict threshold _004") {
sql("drop table if exists local1")
sql(
"""
@@ -212,8 +207,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test("test local dictionary custom configurations for local dict threshold _005")
- {
+ test("test local dictionary custom configurations for local dict threshold _005") {
sql("drop table if exists local1")
sql(
"""
@@ -229,8 +223,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
test("test local dictionary custom configurations with both columns and threshold configured " +
- "_001")
- {
+ "_001") {
sql("drop table if exists local1")
sql(
"""
@@ -252,8 +245,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
test("test local dictionary custom configurations with both columns and threshold configured " +
- "_002")
- {
+ "_002") {
sql("drop table if exists local1")
sql(
"""
@@ -275,8 +267,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
test("test local dictionary custom configurations with both columns and threshold configured " +
- "_003")
- {
+ "_003") {
sql("drop table if exists local1")
sql(
"""
@@ -298,8 +289,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
test("test local dictionary custom configurations with both columns and threshold configured " +
- "_004")
- {
+ "_004") {
sql("drop table if exists local1")
sql(
"""
@@ -335,8 +325,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
test("test local dictionary custom configurations with both columns and threshold configured " +
- "_006")
- {
+ "_006") {
sql("drop table if exists local1")
intercept[MalformedCarbonCommandException] {
sql(
@@ -349,8 +338,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
test("test local dictionary custom configurations with both columns and threshold configured " +
- "_007")
- {
+ "_007") {
sql("drop table if exists local1")
intercept[MalformedCarbonCommandException] {
sql(
@@ -363,8 +351,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
test("test local dictionary custom configurations with both columns and threshold configured " +
- "_008")
- {
+ "_008") {
sql("drop table if exists local1")
intercept[MalformedCarbonCommandException] {
sql(
@@ -378,8 +365,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
test("test local dictionary custom configurations with both columns and threshold configured " +
- "_009")
- {
+ "_009") {
sql("drop table if exists local1")
intercept[MalformedCarbonCommandException] {
sql(
@@ -392,8 +378,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
test("test local dictionary custom configurations with both columns and threshold configured " +
- "_010")
- {
+ "_010") {
sql("drop table if exists local1")
intercept[MalformedCarbonCommandException] {
sql(
@@ -406,8 +391,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
test("test local dictionary custom configurations with both columns and threshold configured " +
- "_011")
- {
+ "_011") {
sql("drop table if exists local1")
intercept[MalformedCarbonCommandException] {
sql(
@@ -457,8 +441,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test("test local dictionary custom configurations when enabled for local dict columns _002")
- {
+ test("test local dictionary custom configurations when enabled for local dict columns _002") {
sql("drop table if exists local1")
intercept[MalformedCarbonCommandException] {
@@ -519,10 +502,10 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
assert(exception.getMessage
.contains(
- "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex " +
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a string/complex/varchar " +
"datatype column. " +
"LOCAL_DICTIONARY_COLUMN should " +
- "be no dictionary string/complex datatype column"))
+ "be no dictionary string/complex/varchar datatype column"))
}
test("test local dictionary custom configurations when enabled for local dict columns _006") {
@@ -539,9 +522,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when local_dictionary_exclude is configured _001")
- {
+ test("test local dictionary custom configurations when local_dictionary_exclude is " +
+ "configured _001") {
sql("drop table if exists local1")
sql(
"""
@@ -559,9 +541,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when local_dictionary_exclude is configured _002")
- {
+ test("test local dictionary custom configurations when local_dictionary_exclude is " +
+ "configured _002") {
sql("drop table if exists local1")
val exception = intercept[MalformedCarbonCommandException] {
sql(
@@ -577,9 +558,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"Please check the DDL."))
}
- test(
- "test local dictionary custom configurations when local_dictionary_exclude is configured _003")
- {
+ test("test local dictionary custom configurations when local_dictionary_exclude is " +
+ "configured _003") {
sql("drop table if exists local1")
val exception = intercept[MalformedCarbonCommandException] {
sql(
@@ -597,9 +577,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
- test(
- "test local dictionary custom configurations when local_dictionary_exclude is configured _004")
- {
+ test("test local dictionary custom configurations when local_dictionary_exclude is " +
+ "configured _004") {
sql("drop table if exists local1")
val exception1 = intercept[MalformedCarbonCommandException] {
sql(
@@ -616,9 +595,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"Please check the DDL."))
}
- test(
- "test local dictionary custom configurations when local_dictionary_exclude is configured _005")
- {
+ test("test local dictionary custom configurations when local_dictionary_exclude is " +
+ "configured _005") {
sql("drop table if exists local1")
val exception = intercept[MalformedCarbonCommandException] {
sql(
@@ -631,15 +609,14 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
assert(exception.getMessage
.contains(
- "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex " +
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a string/complex/varchar " +
"datatype column. " +
"LOCAL_DICTIONARY_COLUMN should " +
- "be no dictionary string/complex datatype column"))
+ "be no dictionary string/complex/varchar datatype column"))
}
- test(
- "test local dictionary custom configurations when local_dictionary_exclude is configured _006")
- {
+ test("test local dictionary custom configurations when local_dictionary_exclude is " +
+ "configured _006") {
sql("drop table if exists local1")
intercept[MalformedCarbonCommandException] {
sql(
@@ -653,10 +630,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when local_dictionary_include and " +
- "local_dictionary_exclude " +
- "is configured _001") {
+ test("test local dictionary custom configurations when local_dictionary_include and " +
+ "local_dictionary_exclude is configured _001") {
sql("drop table if exists local1")
sql(
"""
@@ -680,10 +655,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when local_dictionary_include and " +
- "local_dictionary_exclude " +
- "is configured _002") {
+ test("test local dictionary custom configurations when local_dictionary_include and " +
+ "local_dictionary_exclude is configured _002") {
sql("drop table if exists local1")
sql(
"""
@@ -708,10 +681,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when local_dictionary_include and " +
- "local_dictionary_exclude " +
- "is configured _003") {
+ test("test local dictionary custom configurations when local_dictionary_include and " +
+ "local_dictionary_exclude is configured _003") {
sql("drop table if exists local1")
sql(
"""
@@ -730,10 +701,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
checkExistence(sql("describe formatted local1"), false, "Local Dictionary Exclude")
}
- test(
- "test local dictionary custom configurations when local_dictionary_include and " +
- "local_dictionary_exclude " +
- "is configured _004") {
+ test("test local dictionary custom configurations when local_dictionary_include and " +
+ "local_dictionary_exclude is configured _004") {
sql("drop table if exists local1")
intercept[MalformedCarbonCommandException] {
sql(
@@ -747,10 +716,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when local_dictionary_include and " +
- "local_dictionary_exclude " +
- "is configured _005") {
+ test("test local dictionary custom configurations when local_dictionary_include and " +
+ "local_dictionary_exclude is configured _005") {
sql("drop table if exists local1")
val exception = intercept[MalformedCarbonCommandException] {
sql(
@@ -768,10 +735,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"LOCAL_DICTIONARY_EXCLUDE. Duplicate columns are not allowed."))
}
- test(
- "test local dictionary custom configurations when local_dictionary_include and " +
- "local_dictionary_exclude " +
- "is configured _006") {
+ test("test local dictionary custom configurations when local_dictionary_include and " +
+ "local_dictionary_exclude is configured _006") {
sql("drop table if exists local1")
sql(
"""
@@ -791,10 +756,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when local_dictionary_include and " +
- "local_dictionary_exclude " +
- "is configured _007") {
+ test("test local dictionary custom configurations when local_dictionary_include and " +
+ "local_dictionary_exclude is configured _007") {
sql("drop table if exists local1")
sql(
"""
@@ -832,8 +795,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test("test local dictionary custom configurations when enabled for local dict threshold _002")
- {
+ test("test local dictionary custom configurations when enabled for local dict threshold _002") {
sql("drop table if exists local1")
sql(
"""
@@ -848,8 +810,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test("test local dictionary custom configurations when enabled for local dict threshold _003")
- {
+ test("test local dictionary custom configurations when enabled for local dict threshold _003") {
sql("drop table if exists local1")
sql(
"""
@@ -864,8 +825,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test("test local dictionary custom configurations when enabled for local dict threshold _004")
- {
+ test("test local dictionary custom configurations when enabled for local dict threshold _004") {
sql("drop table if exists local1")
sql(
"""
@@ -880,8 +840,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test("test local dictionary custom configurations when enabled for local dict threshold _005")
- {
+ test("test local dictionary custom configurations when enabled for local dict threshold _005") {
sql("drop table if exists local1")
sql(
"""
@@ -896,10 +855,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when enabled with both columns and threshold " +
- "configured _001")
- {
+ test("test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _001") {
sql("drop table if exists local1")
sql(
"""
@@ -921,10 +878,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when enabled with both columns and threshold " +
- "configured _002")
- {
+ test("test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _002") {
sql("drop table if exists local1")
sql(
"""
@@ -946,10 +901,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when enabled with both columns and threshold " +
- "configured _003")
- {
+ test("test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _003") {
sql("drop table if exists local1")
sql(
"""
@@ -971,10 +924,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when enabled with both columns and threshold " +
- "configured _004")
- {
+ test("test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _004") {
sql("drop table if exists local1")
sql(
"""
@@ -996,10 +947,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when enabled with both columns and threshold " +
- "configured _005")
- {
+ test("test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _005") {
sql("drop table if exists local1")
intercept[MalformedCarbonCommandException] {
sql(
@@ -1012,10 +961,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when enabled with both columns and threshold " +
- "configured _006")
- {
+ test("test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _006") {
sql("drop table if exists local1")
intercept[MalformedCarbonCommandException] {
sql(
@@ -1028,10 +975,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when enabled with both columns and threshold " +
- "configured _007")
- {
+ test("test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _007") {
sql("drop table if exists local1")
intercept[MalformedCarbonCommandException] {
sql(
@@ -1044,10 +989,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when enabled with both columns and threshold " +
- "configured _008")
- {
+ test("test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _008") {
sql("drop table if exists local1")
intercept[MalformedCarbonCommandException] {
sql(
@@ -1060,10 +1003,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when enabled with both columns and threshold " +
- "configured _009")
- {
+ test("test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _009") {
sql("drop table if exists local1")
intercept[MalformedCarbonCommandException] {
sql(
@@ -1076,10 +1017,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when enabled with both columns and threshold " +
- "configured _010")
- {
+ test("test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _010") {
sql("drop table if exists local1")
intercept[MalformedCarbonCommandException] {
sql(
@@ -1092,10 +1031,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when enabled with both columns and threshold " +
- "configured _011")
- {
+ test("test local dictionary custom configurations when enabled with both columns and threshold " +
+ "configured _011") {
sql("drop table if exists local1")
intercept[MalformedCarbonCommandException] {
sql(
@@ -1140,8 +1077,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test("test local dictionary custom configurations when disabled for local dict columns _002")
- {
+ test("test local dictionary custom configurations when disabled for local dict columns _002") {
sql("drop table if exists local1")
sql(
"""
@@ -1231,8 +1167,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test("test local dictionary custom configurations when disabled for local dict threshold _002")
- {
+ test("test local dictionary custom configurations when disabled for local dict threshold _002") {
sql("drop table if exists local1")
sql(
"""
@@ -1263,8 +1198,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test("test local dictionary custom configurations when disabled for local dict threshold _004")
- {
+ test("test local dictionary custom configurations when disabled for local dict threshold _004") {
sql("drop table if exists local1")
sql(
"""
@@ -1279,8 +1213,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test("test local dictionary custom configurations when disabled for local dict threshold _005")
- {
+ test("test local dictionary custom configurations when disabled for local dict threshold _005") {
sql("drop table if exists local1")
sql(
"""
@@ -1297,8 +1230,7 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
test(
"test local dictionary custom configurations when disabled with both columns and threshold " +
- "configured _001")
- {
+ "configured _001") {
sql("drop table if exists local1")
sql(
"""
@@ -1314,10 +1246,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when disabled with both columns and threshold " +
- "configured _002")
- {
+ test("test local dictionary custom configurations when disabled with both columns and threshold" +
+ " configured _002") {
sql("drop table if exists local1")
sql(
"""
@@ -1333,10 +1263,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when disabled with both columns and threshold " +
- "configured _003")
- {
+ test("test local dictionary custom configurations when disabled with both columns and threshold" +
+ " configured _003") {
sql("drop table if exists local1")
sql(
"""
@@ -1352,10 +1280,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when disabled with both columns and threshold " +
- "configured _004")
- {
+ test("test local dictionary custom configurations when disabled with both columns and threshold" +
+ " configured _004") {
sql("drop table if exists local1")
sql(
"""
@@ -1371,10 +1297,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when disabled with both columns and threshold " +
- "configured _005")
- {
+ test("test local dictionary custom configurations when disabled with both columns and threshold" +
+ " configured _005") {
sql("drop table if exists local1")
sql(
@@ -1390,10 +1314,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when disabled with both columns and threshold " +
- "configured _006")
- {
+ test("test local dictionary custom configurations when disabled with both columns and threshold" +
+ " configured _006") {
sql("drop table if exists local1")
sql(
"""
@@ -1409,10 +1331,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when disabled with both columns and threshold " +
- "configured _007")
- {
+ test("test local dictionary custom configurations when disabled with both columns and threshold" +
+ " configured _007") {
sql("drop table if exists local1")
sql(
"""
@@ -1428,10 +1348,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when disabled with both columns and threshold " +
- "configured _008")
- {
+ test("test local dictionary custom configurations when disabled with both columns and threshold" +
+ " configured _008") {
sql("drop table if exists local1")
sql(
"""
@@ -1447,10 +1365,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when disabled with both columns and threshold " +
- "configured _009")
- {
+ test("test local dictionary custom configurations when disabled with both columns and threshold" +
+ " configured _009") {
sql("drop table if exists local1")
sql(
"""
@@ -1466,10 +1382,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when disabled with both columns and threshold " +
- "configured _010")
- {
+ test("test local dictionary custom configurations when disabled with both columns and threshold" +
+ " configured _010") {
sql("drop table if exists local1")
sql(
"""
@@ -1485,10 +1399,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when disabled with both columns and threshold " +
- "configured _011")
- {
+ test("test local dictionary custom configurations when disabled with both columns and threshold" +
+ " configured _011") {
sql("drop table if exists local1")
sql(
"""
@@ -1610,10 +1522,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test CTAS statements for local dictionary custom configurations when enabled for local dict " +
- "columns _001")
- {
+ test("test CTAS statements for local dictionary custom configurations when enabled for " +
+ "local dict columns _001") {
sql("drop table if exists local")
sql("drop table if exists local1")
sql(
@@ -1637,10 +1547,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test CTAS statements for local dictionary custom configurations when enabled for local dict " +
- "columns _002")
- {
+ test("test CTAS statements for local dictionary custom configurations when enabled for " +
+ "local dict columns _002") {
sql("drop table if exists local")
sql("drop table if exists local1")
@@ -1659,10 +1567,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test CTAS statements for local dictionary custom configurations when enabled for local dict " +
- "columns _003")
- {
+ test("test CTAS statements for local dictionary custom configurations when enabled for " +
+ "local dict columns _003") {
sql("drop table if exists local")
sql("drop table if exists local1")
@@ -1687,9 +1593,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
- test(
- "test CTAS statements for local dictionary custom configurations when enabled for local dict " +
- "columns _004")
+ test("test CTAS statements for local dictionary custom configurations when enabled for " +
+ "local dict columns _004")
{
sql("drop table if exists local")
sql("drop table if exists local1")
@@ -1713,10 +1618,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"Please check the DDL."))
}
- test(
- "test CTAS statements for local dictionary custom configurations when enabled for local dict " +
- "columns _005")
- {
+ test("test CTAS statements for local dictionary custom configurations when enabled for " +
+ "local dict columns _005") {
sql("drop table if exists local")
sql("drop table if exists local1")
sql(
@@ -1735,16 +1638,14 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
assert(exception.getMessage
.contains(
- "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex " +
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a string/complex/varchar " +
"datatype column. " +
"LOCAL_DICTIONARY_COLUMN should " +
- "be no dictionary string/complex datatype column"))
+ "be no dictionary string/complex/varchar datatype column"))
}
- test(
- "test CTAS statements for local dictionary custom configurations when enabled for local dict " +
- "columns _006")
- {
+ test("test CTAS statements for local dictionary custom configurations when enabled for local dict " +
+ "columns _006") {
sql("drop table if exists local")
sql("drop table if exists local1")
sql(
@@ -1763,10 +1664,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test CTAS statements for local dictionary custom configurations when " +
- "local_dictionary_exclude is configured _001")
- {
+ test("test CTAS statements for local dictionary custom configurations when " +
+ "local_dictionary_exclude is configured _001") {
sql("drop table if exists local")
sql("drop table if exists local1")
sql(
@@ -1790,10 +1689,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test CTAS statements for local dictionary custom configurations when " +
- "local_dictionary_exclude is configured _002")
- {
+ test("test CTAS statements for local dictionary custom configurations when " +
+ "local_dictionary_exclude is configured _002") {
sql("drop table if exists local1")
sql("drop table if exists local")
sql(
@@ -1811,10 +1708,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test CTAS statements for local dictionary custom configurations when " +
- "local_dictionary_exclude is configured _003")
- {
+ test("test CTAS statements for local dictionary custom configurations when " +
+ "local_dictionary_exclude is configured _003") {
sql("drop table if exists local1")
sql("drop table if exists local")
sql(
@@ -1837,10 +1732,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"Please check the DDL."))
}
- test(
- "test CTAS statements for local dictionary custom configurations when " +
- "local_dictionary_exclude is configured _004")
- {
+ test("test CTAS statements for local dictionary custom configurations when " +
+ "local_dictionary_exclude is configured _004") {
sql("drop table if exists local1")
sql("drop table if exists local")
sql(
@@ -1863,10 +1756,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"Please check the DDL."))
}
- test(
- "test CTAS statements for local dictionary custom configurations when " +
- "local_dictionary_exclude is configured _005")
- {
+ test("test CTAS statements for local dictionary custom configurations when " +
+ "local_dictionary_exclude is configured _005") {
sql("drop table if exists local1")
sql("drop table if exists local")
sql(
@@ -1885,16 +1776,14 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
assert(exception.getMessage
.contains(
- "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a String/complex " +
+ "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: id is not a string/complex/varchar " +
"datatype column. " +
"LOCAL_DICTIONARY_COLUMN should " +
- "be no dictionary string/complex datatype column"))
+ "be no dictionary string/complex/varchar datatype column"))
}
- test(
- "test CTAS statements for local dictionary custom configurations when " +
- "local_dictionary_exclude is configured _006")
- {
+ test("test CTAS statements for local dictionary custom configurations when " +
+ "local_dictionary_exclude is configured _006") {
sql("drop table if exists local1")
sql("drop table if exists local")
sql(
@@ -1913,10 +1802,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test CTAS statements for local dictionary custom configurations when " +
- "local_dictionary_include and local_dictionary_exclude " +
- "is configured _001") {
+ test("test CTAS statements for local dictionary custom configurations when " +
+ "local_dictionary_include and local_dictionary_exclude is configured _001") {
sql("drop table if exists local1")
sql("drop table if exists local")
sql(
@@ -1945,10 +1832,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test CTAS statements for local dictionary custom configurations when " +
- "local_dictionary_include and local_dictionary_exclude " +
- "is configured _002") {
+ test("test CTAS statements for local dictionary custom configurations when " +
+ "local_dictionary_include and local_dictionary_exclude is configured _002") {
sql("drop table if exists local1")
sql("drop table if exists local")
sql(
@@ -1972,10 +1857,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
checkExistence(sql("describe formatted local1"), false, "Local Dictionary Exclude")
}
- test(
- "test CTAS statements for local dictionary custom configurations when " +
- "local_dictionary_include and local_dictionary_exclude " +
- "is configured _003") {
+ test("test CTAS statements for local dictionary custom configurations when " +
+ "local_dictionary_include and local_dictionary_exclude is configured _003") {
sql("drop table if exists local1")
sql("drop table if exists local")
sql(
@@ -1995,10 +1878,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test CTAS statements for local dictionary custom configurations when " +
- "local_dictionary_include and local_dictionary_exclude " +
- "is configured _004") {
+ test("test CTAS statements for local dictionary custom configurations when " +
+ "local_dictionary_include and local_dictionary_exclude is configured _004") {
sql("drop table if exists local1")
sql("drop table if exists local")
sql(
@@ -2017,10 +1898,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test CTAS statements for local dictionary custom configurations when " +
- "local_dictionary_include and local_dictionary_exclude " +
- "is configured _005") {
+ test("test CTAS statements for local dictionary custom configurations when " +
+ "local_dictionary_include and local_dictionary_exclude is configured _005") {
sql("drop table if exists local1")
sql("drop table if exists local")
sql(
@@ -2042,10 +1921,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test CTAS statements for local dictionary custom configurations when enabled for local dict " +
- "threshold _001")
- {
+ test("test CTAS statements for local dictionary custom configurations when enabled for local dict " +
+ "threshold _001") {
sql("drop table if exists local1")
sql("drop table if exists local")
sql(
@@ -2069,10 +1946,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test CTAS statements for local dictionary custom configurations when enabled for local dict " +
- "threshold _002")
- {
+ test("test CTAS statements for local dictionary custom configurations when enabled for local dict " +
+ "threshold _002") {
sql("drop table if exists local1")
sql("drop table if exists local")
sql(
@@ -2093,10 +1968,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test CTAS statements for local dictionary custom configurations when enabled for local dict " +
- "threshold _003")
- {
+ test("test CTAS statements for local dictionary custom configurations when enabled for local dict " +
+ "threshold _003") {
sql("drop table if exists local1")
sql("drop table if exists local")
sql(
@@ -2118,10 +1991,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test CTAS statements for local dictionary custom configurations when first table is hive " +
- "table")
- {
+ test("test CTAS statements for local dictionary custom configurations when first table is hive " +
+ "table") {
sql("drop table if exists local1")
sql("drop table if exists local")
sql(
@@ -2172,9 +2043,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when complex dataType columns are given in " +
- "local_dictionary_include _001") {
+ test("test local dictionary custom configurations when complex dataType columns are given in " +
+ "local_dictionary_include _001") {
sql("drop table if exists local1")
val exception = intercept[MalformedCarbonCommandException] {
sql(
@@ -2190,9 +2060,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"local_dictionary_include are not of string dataType."))
}
- test(
- "test local dictionary custom configurations when complex dataType columns are given in " +
- "local_dictionary_include _002") {
+ test("test local dictionary custom configurations when complex dataType columns are given in " +
+ "local_dictionary_include _002") {
sql("drop table if exists local1")
val exception = intercept[MalformedCarbonCommandException] {
sql(
@@ -2209,10 +2078,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"local_dictionary_include are not of string dataType."))
}
- test(
- "test local dictionary custom configurations when complex dataType columns are given in " +
- "local_dictionary_include _003")
- {
+ test("test local dictionary custom configurations when complex dataType columns are given in " +
+ "local_dictionary_include _003") {
sql("drop table if exists local1")
sql(
"""
@@ -2232,10 +2099,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when complex dataType columns are given in " +
- "local_dictionary_include _004")
- {
+ test("test local dictionary custom configurations when complex dataType columns are given in " +
+ "local_dictionary_include _004") {
sql("drop table if exists local1")
sql(
"""
@@ -2255,10 +2120,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when complex dataType columns are given in " +
- "local_dictionary_include _005")
- {
+ test("test local dictionary custom configurations when complex dataType columns are given in " +
+ "local_dictionary_include _005") {
sql("drop table if exists local1")
sql(
"""
@@ -2278,9 +2141,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when complex dataType columns are given in " +
- "local_dictionary_include _006") {
+ test("test local dictionary custom configurations when complex dataType columns are given in " +
+ "local_dictionary_include _006") {
sql("drop table if exists local1")
sql(
"""
@@ -2301,10 +2163,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when complex dataType columns are given in " +
- "local_dictionary_include _007")
- {
+ test("test local dictionary custom configurations when complex dataType columns are given in " +
+ "local_dictionary_include _007") {
sql("drop table if exists local1")
sql(
"""
@@ -2324,9 +2184,8 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
}
}
- test(
- "test local dictionary custom configurations when complex dataType columns are given in " +
- "local_dictionary_include _008") {
+ test("test local dictionary custom configurations when complex dataType columns are given in " +
+ "local_dictionary_include _008") {
sql("drop table if exists local1")
val exception = intercept[MalformedCarbonCommandException] {
sql(
@@ -2343,6 +2202,96 @@ class LocalDictionarySupportCreateTableTest extends QueryTest with BeforeAndAfte
"local_dictionary_include are not of string dataType."))
}
+ test("test local dictionary custom configurations when varchar columns are specified _001") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string,city string, st array<struct<si:string>>)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('LONG_STRING_COLUMNS'='name','local_dictionary_enable'='true','local_dictionary_include'='name')
+ """.stripMargin)
+ sql("desc formatted local1").show(30,false)
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("10000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("name") && !row.get(1).toString.contains("city"))
+ }
+ }
+
+ test("test local dictionary custom configurations when varchar columns are specified _002") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string,city string, st array<struct<si:string,sd:int>>)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('LONG_STRING_COLUMNS'=' Name','local_dictionary_enable'='true','local_dictionary_include'='name,st')
+ """.stripMargin)
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("10000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("st.val.si,name") && !row.get(1).toString.contains("city"))
+ }
+ }
+
+ test("test local dictionary custom configurations when varchar columns are specified _003") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string,city string, st array<struct<si:string,sd:int>>)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('LONG_STRING_COLUMNS'='name','local_dictionary_enable'='true','local_dictionary_exclude'='name,st')
+ """.stripMargin)
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("10000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Exclude")) match {
+ case Some(row) => assert(row.get(1).toString.contains("st.val.si,name") &&
+ !row.get(1).toString.contains("city"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ }
+
+ test("test local dictionary custom configurations when varchar columns are specified _004") {
+ sql("drop table if exists local1")
+ sql(
+ """
+ | CREATE TABLE local1(id int, name string,city string, st array<struct<si:string,sd:int>>)
+ | STORED BY 'org.apache.carbondata.format'
+ | tblproperties('LONG_STRING_COLUMNS'=' Name','local_dictionary_enable'='true',
+ | 'local_dictionary_include'='name,st','local_dictionary_exclude'='city')
+ """.stripMargin)
+ val descLoc = sql("describe formatted local1").collect
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Threshold")) match {
+ case Some(row) => assert(row.get(1).toString.contains("10000"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Enabled")) match {
+ case Some(row) => assert(row.get(1).toString.contains("true"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Include")) match {
+ case Some(row) => assert(row.get(1).toString.contains("st.val.si,name") &&
+ !row.get(1).toString.contains("city"))
+ }
+ descLoc.find(_.get(0).toString.contains("Local Dictionary Exclude")) match {
+ case Some(row) => assert(row.get(1).toString.contains("city"))
+ }
+ }
+
override protected def afterAll(): Unit = {
sql("DROP TABLE IF EXISTS LOCAL1")
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f0c77e5f/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 84fd25f..d705fc9 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -457,17 +457,19 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
}
}
- // check if column is other than string datatype
+ // check if column is other than STRING or VARCHAR datatype
localDictColumns.foreach { dictColm =>
if (fields
.exists(x => x.column.equalsIgnoreCase(dictColm) &&
!x.dataType.get.equalsIgnoreCase("STRING") &&
+ !x.dataType.get.equalsIgnoreCase("VARCHAR") &&
!x.dataType.get.equalsIgnoreCase("STRUCT") &&
!x.dataType.get.equalsIgnoreCase("ARRAY"))) {
val errormsg = "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: " +
dictColm.trim +
- " is not a String/complex datatype column. LOCAL_DICTIONARY_COLUMN should " +
- "be no dictionary string/complex datatype column.Please check the DDL."
+ " is not a string/complex/varchar datatype column. LOCAL_DICTIONARY_COLUMN" +
+ " should be no dictionary string/complex/varchar datatype column." +
+ "Please check the DDL."
throw new MalformedCarbonCommandException(errormsg)
}
}
@@ -685,12 +687,12 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
noInvertedIdxColsProps =
tableProperties.get(CarbonCommonConstants.NO_INVERTED_INDEX).get.split(',').map(_.trim)
noInvertedIdxColsProps.foreach { noInvertedIdxColProp =>
- if (!fields.exists(x => x.column.equalsIgnoreCase(noInvertedIdxColProp))) {
- val errormsg = "NO_INVERTED_INDEX column: " + noInvertedIdxColProp +
- " does not exist in table. Please check create table statement."
- throw new MalformedCarbonCommandException(errormsg)
- }
+ if (!fields.exists(x => x.column.equalsIgnoreCase(noInvertedIdxColProp))) {
+ val errormsg = "NO_INVERTED_INDEX column: " + noInvertedIdxColProp +
+ " does not exist in table. Please check create table statement."
+ throw new MalformedCarbonCommandException(errormsg)
}
+ }
}
// check duplicate columns and only 1 col left
val distinctCols = noInvertedIdxColsProps.toSet
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f0c77e5f/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
index 8a014ce..6fee8c7 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchemaCommon.scala
@@ -328,6 +328,7 @@ class AlterTableColumnSchemaGenerator(
includeColumns.append(column)
} else {
if (column.getDataType.equals(DataTypes.STRING) ||
+ column.getDataType.equals(DataTypes.VARCHAR) ||
column.getDataType.toString.equals("ARRAY") ||
column.getDataType.toString.equals("STRUCT")) {
excludeColumns.append(column)
@@ -348,6 +349,7 @@ class AlterTableColumnSchemaGenerator(
if (null == localDictExcludeColumns) {
columns.foreach { column =>
if (column.getDataType.equals(DataTypes.STRING) ||
+ column.getDataType.equals(DataTypes.VARCHAR) ||
column.getDataType.toString.equals("ARRAY") ||
column.getDataType.toString.equals("STRUCT")) {
includeColumns.append(column)
@@ -357,6 +359,7 @@ class AlterTableColumnSchemaGenerator(
columns.foreach { column =>
if (!localDictExcludeColumns.contains(column.getColumnName) &&
(column.getDataType.equals(DataTypes.STRING) ||
+ column.getDataType.equals(DataTypes.VARCHAR) ||
column.getDataType.toString.equals("ARRAY") ||
column.getDataType.toString.equals("STRUCT"))) {
includeColumns.append(column)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/f0c77e5f/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
index 4659c39..b5450b3 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
@@ -42,7 +42,7 @@ import org.apache.carbondata.core.metadata.schema.table.CarbonTable
import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema
import org.apache.carbondata.core.util.path.CarbonTablePath
import org.apache.carbondata.core.util.CarbonUtil
-import org.apache.carbondata.format.{SchemaEvolutionEntry, TableInfo}
+import org.apache.carbondata.format.{Encoding, SchemaEvolutionEntry, TableInfo}
import org.apache.carbondata.spark.util.{CarbonScalaUtil, CommonUtil}
@@ -691,62 +691,68 @@ object AlterTableUtil {
}
/**
- * Verify if specified column is of no-dictionary string dataType
+ * Verify if specified column is of no-dictionary string or varchar dataType
*/
localDictColumns.foreach { dictCol =>
if (allColumns.exists(col => col.getColumnName.equalsIgnoreCase(dictCol) &&
!col.getDataType.toString
.equalsIgnoreCase("STRING") &&
!col.getDataType.toString
+ .equalsIgnoreCase("VARCHAR") &&
+ !col.getDataType.toString
.equalsIgnoreCase("STRUCT") &&
!col.getDataType.toString
.equalsIgnoreCase("ARRAY"))) {
val errMsg = "LOCAL_DICTIONARY_INCLUDE/LOCAL_DICTIONARY_EXCLUDE column: " + dictCol.trim +
- " is not a String/complex datatype column. LOCAL_DICTIONARY_INCLUDE" +
+ " is not a string/complex/varchar datatype column. LOCAL_DICTIONARY_INCLUDE" +
"/LOCAL_DICTIONARY_EXCLUDE should be no " +
- "dictionary string/complex datatype column."
+ "dictionary string/complex/varchar datatype column."
throw new MalformedCarbonCommandException(errMsg)
}
}
-
- // Validate whether any of the child columns of complex dataType column is a string column
- localDictColumns.foreach { dictColm =>
- if (allColumns
- .exists(x => x.getColumnName.equalsIgnoreCase(dictColm) && x.getNumberOfChild > 0 &&
- !validateChildColumns(allColumns, dictColm))) {
- val errMsg = "None of the child columns specified in the complex dataType column(s) in " +
- "local_dictionary_include are not of string dataType."
- throw new MalformedCarbonCommandException(errMsg)
+ var countOfDictCols = 0
+ var trav = 0
+ // Validate whether any of the child columns of complex dataType column is a string or
+ // varchar dataType column
+ if (property._1.equalsIgnoreCase(CarbonCommonConstants.LOCAL_DICTIONARY_INCLUDE)) {
+ // Validate whether any of the child columns of complex dataType column is a string column
+ localDictColumns.foreach { dictColm =>
+ for (elem <- allColumns.indices) {
+ var column = allColumns(elem)
+ if (column.getColumnName.equalsIgnoreCase(dictColm) && column.getNumberOfChild > 0 &&
+ !validateChildColumns(allColumns, column.getNumberOfChild, elem. +(1))) {
+ val errMsg =
+ "None of the child columns specified in the complex dataType column(s) in " +
+ "local_dictionary_include are not of string dataType."
+ throw new MalformedCarbonCommandException(errMsg)
+ }
+ }
}
}
/**
- * check whether any child column present in complex type column is string type
+ * check whether any child column present in complex type column is string or varchar type
*
* @param schemas
* @return
*/
def validateChildColumns(schemas: mutable.Buffer[ColumnSchema],
- complexColumn: String): Boolean = {
- var childColumnCount = 0
- var numberOfPrimitiveColumns = 0
- schemas.foreach { column =>
- if (childColumnCount > 0) {
- if (column.getDataType.equals(DataTypes.STRING)) {
- primitiveComplexChildColumns.add(column.getColumnName)
- numberOfPrimitiveColumns += 1
- childColumnCount -= 1
- } else {
- childColumnCount -= 1
+ colCount: Int, traverse: Int): Boolean = {
+ trav = traverse
+ var column: ColumnSchema = null
+ for (i <- 0 until colCount) {
+ column = schemas(trav)
+ if (column.getNumberOfChild > 0) {
+ validateChildColumns(schemas, column.getNumberOfChild, trav. +(1))
+ } else {
+ if (column.isDimensionColumn && (column.getDataType.equals(DataTypes.STRING) ||
+ column.getDataType.equals(DataTypes.VARCHAR))) {
+ countOfDictCols += 1
}
- }
- if ((localDictColumns.exists(x => x.equalsIgnoreCase(column.getColumnName)) ||
- primitiveComplexChildColumns.contains(column.getColumnName)) &&
- column.getNumberOfChild > 0) {
- childColumnCount = column.getNumberOfChild
+ trav = trav + 1
}
}
- if (numberOfPrimitiveColumns > 0) {
+ if (countOfDictCols > 0) {
return true
}
false