You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/06/22 01:34:49 UTC
[48/50] [abbrv] carbondata git commit:
[CARBONDATA-2509][CARBONDATA-2510][CARBONDATA-2511][32K] Add validate for
long string columns
[CARBONDATA-2509][CARBONDATA-2510][CARBONDATA-2511][32K] Add validate for long string columns
Add validate for long string columns
1. long string columns cannot be sort_columns
2. long string columns cannot be dictionary include
3. long string columns cannot be dictionary exclude
4. long string columns can only be string columns
This closes #2380
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/218a8deb
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/218a8deb
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/218a8deb
Branch: refs/heads/carbonstore
Commit: 218a8deb614e0dc160ab8c7e38c71d80711eb1a7
Parents: 091a28b
Author: xuchuanyin <xu...@hust.edu.cn>
Authored: Wed Jun 20 18:55:04 2018 +0800
Committer: kumarvishal09 <ku...@gmail.com>
Committed: Thu Jun 21 12:00:34 2018 +0530
----------------------------------------------------------------------
.../VarcharDataTypesBasicTestCase.scala | 57 ++++++++++++++++++++
.../spark/sql/catalyst/CarbonDDLSqlParser.scala | 11 +++-
2 files changed, 67 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/218a8deb/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
index 419b306..9ea3f1f 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/longstring/VarcharDataTypesBasicTestCase.scala
@@ -74,6 +74,63 @@ class VarcharDataTypesBasicTestCase extends QueryTest with BeforeAndAfterEach wi
sql(s"drop table if exists $longStringTable")
}
+ test("long string columns cannot be dictionary include") {
+ val exceptionCaught = intercept[Exception] {
+ sql(
+ s"""
+ | CREATE TABLE if not exists $longStringTable(
+ | id INT, name STRING, description STRING, address STRING, note STRING
+ | ) STORED BY 'carbondata'
+ | TBLPROPERTIES('LONG_STRING_COLUMNS'='address, note', 'dictionary_include'='address')
+ |""".
+ stripMargin)
+ }
+ assert(exceptionCaught.getMessage.contains("DICTIONARY_INCLUDE is unsupported for long string datatype column: address"))
+ }
+
+ test("long string columns cannot be dictionay exclude") {
+ val exceptionCaught = intercept[Exception] {
+ sql(
+ s"""
+ | CREATE TABLE if not exists $longStringTable(
+ | id INT, name STRING, description STRING, address STRING, note STRING
+ | ) STORED BY 'carbondata'
+ | TBLPROPERTIES('LONG_STRING_COLUMNS'='address, note', 'dictionary_exclude'='address')
+ |""".
+ stripMargin)
+ }
+ assert(exceptionCaught.getMessage.contains("DICTIONARY_EXCLUDE is unsupported for long string datatype column: address"))
+ }
+
+ test("long string columns cannot be sort_columns") {
+ val exceptionCaught = intercept[Exception] {
+ sql(
+ s"""
+ | CREATE TABLE if not exists $longStringTable(
+ | id INT, name STRING, description STRING, address STRING, note STRING
+ | ) STORED BY 'carbondata'
+ | TBLPROPERTIES('LONG_STRING_COLUMNS'='name, note', 'SORT_COLUMNS'='name, address')
+ |""".
+ stripMargin)
+ }
+ assert(exceptionCaught.getMessage.contains("sort_columns is unsupported for long string datatype column: name"))
+ }
+
+ test("long string columns can only be string columns") {
+ val exceptionCaught = intercept[Exception] {
+ sql(
+ s"""
+ | CREATE TABLE if not exists $longStringTable(
+ | id INT, name STRING, description STRING, address STRING, note STRING
+ | ) STORED BY 'carbondata'
+ | TBLPROPERTIES('LONG_STRING_COLUMNS'='id, note')
+ |""".
+ stripMargin)
+ }
+ assert(exceptionCaught.getMessage.contains("long_string_columns: id"))
+ assert(exceptionCaught.getMessage.contains("its data type is not string"))
+ }
+
private def prepareTable(): Unit = {
sql(
s"""
http://git-wip-us.apache.org/repos/asf/carbondata/blob/218a8deb/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
index 350fc36..7c3cf65 100644
--- a/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
+++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/catalyst/CarbonDDLSqlParser.scala
@@ -752,7 +752,7 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
}
if (varcharCols.exists(x => x.equalsIgnoreCase(column))) {
throw new MalformedCarbonCommandException(
- s"sort_columns is unsupported for long string datatype column $column")
+ s"sort_columns is unsupported for long string datatype column: $column")
}
}
}
@@ -791,6 +791,10 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
val errorMsg = "DICTIONARY_EXCLUDE is unsupported for " + dataType.toLowerCase() +
" data type column: " + dictExcludeCol
throw new MalformedCarbonCommandException(errorMsg)
+ } else if (varcharCols.exists(x => x.equalsIgnoreCase(dictExcludeCol))) {
+ throw new MalformedCarbonCommandException(
+ "DICTIONARY_EXCLUDE is unsupported for long string datatype column: " +
+ dictExcludeCol)
}
}
}
@@ -805,6 +809,11 @@ abstract class CarbonDDLSqlParser extends AbstractCarbonSparkSQLParser {
" does not exist in table. Please check create table statement."
throw new MalformedCarbonCommandException(errormsg)
}
+ if (varcharCols.exists(x => x.equalsIgnoreCase(distIncludeCol.trim))) {
+ throw new MalformedCarbonCommandException(
+ "DICTIONARY_INCLUDE is unsupported for long string datatype column: " +
+ distIncludeCol.trim)
+ }
}
}