You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by in...@apache.org on 2021/05/24 07:02:29 UTC
[carbondata] branch master updated: [CARBONDATA-4184] alter table
Set TBLPROPERTIES for RANGE_COLUMN sets unsupported
datatype(complex_datatypes/Binary/Boolean/Decimal) as RANGE_COLUMN
This is an automated email from the ASF dual-hosted git repository.
indhumuthumurugesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 35091a2 [CARBONDATA-4184] alter table Set TBLPROPERTIES for RANGE_COLUMN sets unsupported datatype(complex_datatypes/Binary/Boolean/Decimal) as RANGE_COLUMN
35091a2 is described below
commit 35091a2e5c837b9ecdb292ab70038b8e329813db
Author: Karan980 <ka...@gmail.com>
AuthorDate: Wed May 12 17:04:53 2021 +0530
[CARBONDATA-4184] alter table Set TBLPROPERTIES for RANGE_COLUMN sets unsupported
datatype(complex_datatypes/Binary/Boolean/Decimal) as RANGE_COLUMN
Why is this PR needed?
Alter table set command was not validating unsupported dataTypes for range column.
What changes were proposed in this PR?
Added validation for unsupported dataTypes before setting range column value.
Does this PR introduce any user interface change?
No
Is any new testcase added?
Yes
This closes #4133
---
.../spark/sql/catalyst/CarbonParserUtil.scala | 16 ++++++++------
.../org/apache/spark/util/AlterTableUtil.scala | 7 +++++-
.../dataload/TestRangeColumnDataLoad.scala | 25 ++++++++++++++++++++++
3 files changed, 41 insertions(+), 7 deletions(-)
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/catalyst/CarbonParserUtil.scala b/integration/spark/src/main/scala/org/apache/spark/sql/catalyst/CarbonParserUtil.scala
index 9cb8ed7..266cd95 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/catalyst/CarbonParserUtil.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/catalyst/CarbonParserUtil.scala
@@ -589,6 +589,15 @@ object CarbonParserUtil {
noInvertedIdxCols
}
+ def validateUnsupportedDataTypeForRangeColumn(dataType: String): Boolean = {
+ DataTypes.BINARY.getName.equalsIgnoreCase(dataType) ||
+ DataTypes.BOOLEAN.getName.equalsIgnoreCase(dataType) ||
+ CarbonCommonConstants.ARRAY.equalsIgnoreCase(dataType) ||
+ CarbonCommonConstants.STRUCT.equalsIgnoreCase(dataType) ||
+ CarbonCommonConstants.MAP.equalsIgnoreCase(dataType) ||
+ CarbonCommonConstants.DECIMAL.equalsIgnoreCase(dataType)
+ }
+
protected def extractInvertedIndexColumns(fields: Seq[Field],
tableProperties: Map[String, String]): Seq[String] = {
// check whether the column name is in fields
@@ -684,12 +693,7 @@ object CarbonParserUtil {
val errorMsg = "range_column: " + rangeColumn +
" does not exist in table. Please check the create table statement."
throw new MalformedCarbonCommandException(errorMsg)
- } else if (DataTypes.BINARY.getName.equalsIgnoreCase(dataType) ||
- DataTypes.BOOLEAN.getName.equalsIgnoreCase(dataType) ||
- CarbonCommonConstants.ARRAY.equalsIgnoreCase(dataType) ||
- CarbonCommonConstants.STRUCT.equalsIgnoreCase(dataType) ||
- CarbonCommonConstants.MAP.equalsIgnoreCase(dataType) ||
- CarbonCommonConstants.DECIMAL.equalsIgnoreCase(dataType)) {
+ } else if (validateUnsupportedDataTypeForRangeColumn(dataType)) {
throw new MalformedCarbonCommandException(
s"RANGE_COLUMN doesn't support $dataType data type: " + rangeColumn)
} else {
diff --git a/integration/spark/src/main/scala/org/apache/spark/util/AlterTableUtil.scala b/integration/spark/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
index c6eca1c..2133b16 100644
--- a/integration/spark/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/util/AlterTableUtil.scala
@@ -25,7 +25,7 @@ import scala.collection.mutable.ListBuffer
import org.apache.commons.lang3.StringUtils
import org.apache.spark.sql.{CarbonEnv, SparkSession}
-import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.{CarbonParserUtil, TableIdentifier}
import org.apache.spark.sql.catalyst.catalog.SessionCatalog
import org.apache.spark.sql.hive.{CarbonRelation, CarbonSessionCatalogUtil}
import org.apache.spark.sql.index.CarbonIndexUtil
@@ -652,6 +652,11 @@ object AlterTableUtil {
throw new MalformedCarbonCommandException(
s"Table property ${ CarbonCommonConstants.RANGE_COLUMN }: ${ rangeColumnProp }" +
s" is not exists in the table")
+ }
+ val dataType = rangeColumn.getDataType.getName;
+ if (CarbonParserUtil.validateUnsupportedDataTypeForRangeColumn(dataType)) {
+ throw new MalformedCarbonCommandException(
+ s"RANGE_COLUMN doesn't support $dataType data type: " + rangeColumnProp)
} else {
propertiesMap.put(CarbonCommonConstants.RANGE_COLUMN, rangeColumn.getColName)
}
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestRangeColumnDataLoad.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestRangeColumnDataLoad.scala
index b8dc63c..f426afb 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestRangeColumnDataLoad.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestRangeColumnDataLoad.scala
@@ -847,6 +847,31 @@ class TestRangeColumnDataLoad extends QueryTest with BeforeAndAfterEach with Bef
sql("ALTER TABLE carbon_range_column6 SET TBLPROPERTIES('range_column'='name')")
}
+ test("check range column validation for unsupported data types") {
+ sql("""drop table if exists carbon_range_column7""".stripMargin)
+ sql(
+ """
+ | CREATE TABLE carbon_range_column7(id INT, name STRING, city STRING, age INT, bin binary,
+ | bool1 boolean, arr1 array<int>, struct1 struct<id1:string,name1:string>,
+ | map1 map<string,string>, dec1 decimal(10,5)) STORED AS carbondata
+ | TBLPROPERTIES('SORT_SCOPE'='LOCAL_SORT', 'SORT_COLUMNS'='name, city')
+ """.stripMargin)
+ val invalidRangeColumnsMap = Map("bin" -> "BINARY", "bool1" -> "BOOLEAN", "arr1" -> "ARRAY",
+ "struct1" -> "STRUCT", "map1" -> "MAP", "dec1" -> "DECIMAL")
+ validateInvalidRangeColumn(invalidRangeColumnsMap)
+ sql("""drop table if exists carbon_range_column7""".stripMargin)
+ }
+
+ def validateInvalidRangeColumn(invalidRangeColumnsMap: Map[String, String]): Unit = {
+ invalidRangeColumnsMap.foreach { value =>
+ val ex = intercept[RuntimeException] {
+ sql(s"ALTER TABLE carbon_range_column7 SET TBLPROPERTIES('range_column'='${ value._1 }')")
+ }
+ assertResult("Alter table newProperties operation failed: RANGE_COLUMN doesn't " +
+ s"support ${ value._2 } data type: ${value._1}")(ex.getMessage)
+ }
+ }
+
private def getIndexFileCount(tableName: String, segmentNo: String = "0"): Int = {
val carbonTable = CarbonMetadata.getInstance().getCarbonTable("default", tableName)
val segmentDir = CarbonTablePath.getSegmentPath(carbonTable.getTablePath, segmentNo)