You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ku...@apache.org on 2019/06/13 14:23:47 UTC
[carbondata] branch master updated: [CARBONDATA-3424] Carbon should
throw proper exception when Select query with average function for
substring of binary column
This is an automated email from the ASF dual-hosted git repository.
kunalkapoor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git
The following commit(s) were added to refs/heads/master by this push:
new 25ffdd3 [CARBONDATA-3424] Carbon should throw proper exception when Select query with average function for substring of binary column
25ffdd3 is described below
commit 25ffdd3bd254dea16c761d1d97112f1e89d47472
Author: xubo245 <xu...@huawei.com>
AuthorDate: Wed Jun 12 15:13:46 2019 +0800
[CARBONDATA-3424] Carbon should throw proper exception when Select query
with average function for substring of binary column
Carbon should throw proper exception when Select query with
average function for substring of binary column
This closes #3272
---
.../testsuite/binary/TestBinaryDataType.scala | 78 ++++++++++++++++++++++
.../spark/sql/hive/CarbonPreAggregateRules.scala | 5 +-
2 files changed, 82 insertions(+), 1 deletion(-)
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/binary/TestBinaryDataType.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/binary/TestBinaryDataType.scala
index 1b73aba..390efe9 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/binary/TestBinaryDataType.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/binary/TestBinaryDataType.scala
@@ -1580,6 +1580,84 @@ class TestBinaryDataType extends QueryTest with BeforeAndAfterAll {
}
}
+ test("Select query with average function for substring of binary column is executed.") {
+ sql("DROP TABLE IF EXISTS uniqdata")
+ sql(
+ s"""
+ | CREATE TABLE uniqdata (
+ | CUST_ID int,
+ | CUST_NAME binary,
+ | ACTIVE_EMUI_VERSION string,
+ | DOB timestamp,
+ | DOJ timestamp,
+ | BIGINT_COLUMN1 bigint,
+ | BIGINT_COLUMN2 bigint,
+ | DECIMAL_COLUMN1 decimal(30,10),
+ | DECIMAL_COLUMN2 decimal(36,10),
+ | Double_COLUMN1 double,
+ | Double_COLUMN2 double,
+ | INTEGER_COLUMN1 int)
+ | STORED BY 'org.apache.carbondata.format'
+ | TBLPROPERTIES('table_blocksize'='2000')
+ """.stripMargin)
+ sql(
+ s"""
+ | LOAD DATA inpath '$resourcesPath/restructure/data_2000.csv'
+ | into table uniqdata
+ | OPTIONS(
+ | 'DELIMITER'=',' ,
+ | 'QUOTECHAR'='"',
+ | 'BAD_RECORDS_ACTION'='FORCE',
+ | 'FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')
+ """.stripMargin)
+
+ sql(
+ s"""select substr(CUST_NAME,1,2)
+ | from uniqdata
+ """.stripMargin).show()
+
+ val e1 = intercept[Exception] {
+ sql(
+ s"""select avg(substr(CUST_NAME,1,2))
+ | from uniqdata
+ """.stripMargin).show()
+ }
+ assert(e1.getMessage.contains("cannot resolve 'avg(substring(uniqdata.`CUST_NAME`, 1, 2))' due to data type mismatch: function average requires numeric types, not BinaryType"))
+
+ val e2 = intercept[Exception] {
+ sql(
+ s"""
+ | select
+ | max(substr(CUST_NAME,1,2)),
+ | min(substr(CUST_NAME,1,2)),
+ | avg(substr(CUST_NAME,1,2)),
+ | count(substr(CUST_NAME,1,2)),
+ | sum(substr(CUST_NAME,1,2)),
+ | variance(substr(CUST_NAME,1,2))
+ | from uniqdata
+ | where CUST_ID IS NULL or DOB IS NOT NULL or BIGINT_COLUMN1 =1233720368578 or DECIMAL_COLUMN1 = 12345678901.1234000058 or Double_COLUMN1 = 1.12345674897976E10 or INTEGER_COLUMN1 IS NULL limit 10
+ """.stripMargin)
+ }
+ assert(e2.getMessage.contains("cannot resolve 'avg(substring(uniqdata.`CUST_NAME`, 1, 2))' due to data type mismatch: function average requires numeric types, not BinaryType"))
+
+ val e3 = intercept[Exception] {
+ sql(
+ s"""
+ | select
+ | max(substring(CUST_NAME,1,2)),
+ | min(substring(CUST_NAME,1,2)),
+ | avg(substring(CUST_NAME,1,2)),
+ | count(substring(CUST_NAME,1,2)),
+ | sum(substring(CUST_NAME,1,2)),
+ | variance(substring(CUST_NAME,1,2))
+ | from uniqdata
+ | where CUST_ID IS NULL or DOB IS NOT NULL or BIGINT_COLUMN1 =1233720368578 or DECIMAL_COLUMN1 = 12345678901.1234000058 or Double_COLUMN1 = 1.12345674897976E10 or INTEGER_COLUMN1 IS NULL limit 10
+ """.stripMargin)
+ }
+ assert(e3.getMessage.contains("cannot resolve 'avg(substring(uniqdata.`CUST_NAME`, 1, 2))' due to data type mismatch: function average requires numeric types, not BinaryType"))
+
+ }
+
override def afterAll: Unit = {
sql("DROP TABLE IF EXISTS binaryTable")
sql("DROP TABLE IF EXISTS hiveTable")
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
index 9be0961..d316309 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
@@ -162,7 +162,10 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession) extends Rule
var isValidPlan = true
logicalPlan.transform {
case aggregate@Aggregate(grp, aExp, child) =>
- isValidPlan = !aExp.exists(p => p.name.equals("preAggLoad") || p.name.equals("preAgg"))
+ isValidPlan = !aExp.exists { p =>
+ if (p.isInstanceOf[UnresolvedAlias]) return false
+ p.name.equals("preAggLoad") || p.name.equals("preAgg")
+ }
val updatedAggExp = aExp.filterNot(_.name.equalsIgnoreCase("preAggLoad"))
Aggregate(grp, updatedAggExp, child)
}