You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ak...@apache.org on 2021/06/18 06:59:08 UTC
[carbondata] branch master updated: [CARBONDATA-4208] Wrong
Exception received for complex child long string columns
This is an automated email from the ASF dual-hosted git repository.
akashrn5 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 65fad98 [CARBONDATA-4208] Wrong Exception received for complex child long string columns
65fad98 is described below
commit 65fad9887425a8525a5e61d9c4ed67e3c2e39f84
Author: Mahesh Raju Somalaraju <ma...@huawei.com>
AuthorDate: Sat Jun 12 01:17:27 2021 +0530
[CARBONDATA-4208] Wrong Exception received for complex child long string columns
Why is this PR needed?
When we create a table with complex columns with child columns with long string
data type then receiving column not found in table exception. Normally it should
throw an exception in the above case by saying that complex child columns will
not support long string data type.
What changes were proposed in this PR?
Added a case if complex child column has long string data type then throw correct
exception.
Exception: MalformedCarbonCommandException
Exception Message: Complex child column cannot be set as LONG_STRING_COLUMNS
Does this PR introduce any user interface change?
No
Is any new testcase added?
Yes
This closes #4150
---
.../metadata/schema/table/column/ColumnSchema.java | 4 ++--
.../apache/carbondata/core/util/CarbonUtil.java | 8 ++++++++
.../spark/sql/catalyst/CarbonParserUtil.scala | 4 ++++
.../complexType/TestComplexDataType.scala | 24 ++++++++++++++++++++++
4 files changed, 38 insertions(+), 2 deletions(-)
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
index 88b55ce..29d7ed0 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/column/ColumnSchema.java
@@ -37,6 +37,7 @@ import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.Writable;
import org.apache.carbondata.core.metadata.schema.table.WritableUtil;
import org.apache.carbondata.core.preagg.TimeSeriesUDF;
+import org.apache.carbondata.core.util.CarbonUtil;
/**
* Store the information about the column meta data present the table
@@ -589,8 +590,7 @@ public class ColumnSchema implements Serializable, Writable, Cloneable {
* @return
*/
public boolean isComplexColumn() {
- return this.getColumnName()
- .contains(".val") || this.getColumnName().contains(".");
+ return CarbonUtil.isComplexColumn(this.getColumnName());
}
public ColumnSchema clone() {
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 1b81950..68919ba 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -3500,4 +3500,12 @@ public final class CarbonUtil {
dataOutputStream.write(CarbonCommonConstants.EMPTY_BYTE_ARRAY);
}
}
+
+ /**
+ * returns whether column is complex column based on column name for child column
+ * @return true if column is complex
+ */
+ public static boolean isComplexColumn(String colName) {
+ return colName.contains(".val") || colName.contains(CarbonCommonConstants.POINT);
+ }
}
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/catalyst/CarbonParserUtil.scala b/integration/spark/src/main/scala/org/apache/spark/sql/catalyst/CarbonParserUtil.scala
index f857355..aed63fe 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/catalyst/CarbonParserUtil.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/catalyst/CarbonParserUtil.scala
@@ -449,6 +449,10 @@ object CarbonParserUtil {
var tmpStr: String = ""
varcharCols.foreach {
column =>
+ if (CarbonUtil.isComplexColumn(column)) {
+ val errMsg = s"Complex child column $column cannot be set as LONG_STRING_COLUMNS"
+ throw new MalformedCarbonCommandException(errMsg)
+ }
tmpStr = column.toUpperCase
duplicateColumnErr.get(tmpStr) match {
case None => duplicateColumnErr.put(tmpStr, 1)
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
index 0610b3e..5e736f9 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexDataType.scala
@@ -27,6 +27,7 @@ import org.apache.spark.sql.test.util.QueryTest
import org.scalatest.BeforeAndAfterAll
import org.apache.carbondata.common.constants.LoggerAction
+import org.apache.carbondata.common.exceptions.sql.MalformedCarbonCommandException
import org.apache.carbondata.core.constants.CarbonCommonConstants
import org.apache.carbondata.core.util.CarbonProperties
@@ -765,6 +766,29 @@ class TestComplexDataType extends QueryTest with BeforeAndAfterAll {
assertResult("Unsupported operation on Complex data type")(arrayException.getMessage)
}
+ test("testing the long string properties for complex columns in main table") {
+ sql("drop table if exists complex1")
+ sql("drop table if exists complex2")
+ sql("drop table if exists complex3")
+ sql("create table complex1 (a int, arr1 array<string>) stored as carbondata")
+ assert(intercept[RuntimeException] {
+ sql("alter table complex1 SET TBLPROPERTIES ('LONG_STRING_COLUMNS'='arr1.val')")
+ }.getMessage.contains(
+ "Alter table newProperties operation failed: Complex child column arr1.val cannot be set " +
+ "as LONG_STRING_COLUMNS"))
+ assert(intercept[MalformedCarbonCommandException] {
+ sql("create table complex2 (a int, arr1 array<string>) " +
+ "stored as carbondata TBLPROPERTIES('LONG_STRING_COLUMNS'='arr1.val')")
+ }.getMessage.contains("Complex child column arr1.val cannot be set as LONG_STRING_COLUMNS"))
+ assert(intercept[MalformedCarbonCommandException] {
+ sql("create table complex3 (a int, struct1 struct<b:string,c:string>) " +
+ "stored as carbondata TBLPROPERTIES('LONG_STRING_COLUMNS'='struct1.b')")
+ }.getMessage.contains("Complex child column struct1.b cannot be set as LONG_STRING_COLUMNS"))
+ sql("drop table if exists complex1")
+ sql("drop table if exists complex2")
+ sql("drop table if exists complex3")
+ }
+
test("check update operation on primitive data types when complex type present in table which " +
"has child name equal to primitive data types") {
sql("drop table if exists update_complex")