You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/01/16 15:36:28 UTC
[3/7] carbondata git commit: [CARBONDATA-2029] Fixed Pre Aggregate
table issue with expression
[CARBONDATA-2029] Fixed Pre Aggregate table issue with expression
Query with expression is giving wrong result
This closes #1803
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/04d40ccb
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/04d40ccb
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/04d40ccb
Branch: refs/heads/carbonstore
Commit: 04d40ccb911199bad4825292214c5ba34910f12a
Parents: 0d23461
Author: kumarvishal <ku...@gmail.com>
Authored: Mon Jan 15 16:26:36 2018 +0530
Committer: kunal642 <ku...@gmail.com>
Committed: Tue Jan 16 14:48:22 2018 +0530
----------------------------------------------------------------------
.../TestPreAggregateTableSelection.scala | 5 +++++
.../spark/sql/hive/CarbonPreAggregateRules.scala | 19 ++++++++++++++++---
2 files changed, 21 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/04d40ccb/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
index 9bbba3a..559e91f 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/preaggregate/TestPreAggregateTableSelection.scala
@@ -262,6 +262,11 @@ class TestPreAggregateTableSelection extends QueryTest with BeforeAndAfterAll {
preAggTableValidator(df.queryExecution.analyzed, "maintable_agg0")
}
+ test("Test query with math operation hitting fact table") {
+ val df = sql("select sum(id)+count(id) from maintable")
+ preAggTableValidator(df.queryExecution.analyzed, "maintable")
+ }
+
override def afterAll: Unit = {
sql("drop table if exists mainTable")
sql("drop table if exists lineitem")
http://git-wip-us.apache.org/repos/asf/carbondata/blob/04d40ccb/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
index de554c5..299ed4d 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonPreAggregateRules.scala
@@ -1013,6 +1013,7 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession) extends Rule
carbonTable: CarbonTable,
queryColumns: scala.collection.mutable.HashSet[QueryColumn],
aggreagteExps: scala.collection.mutable.HashSet[AggregateExpression]): Boolean = {
+ var isValid = true
groupByExpression foreach { expression =>
extractColumnFromExpression(expression, queryColumns, carbonTable)
}
@@ -1025,13 +1026,13 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession) extends Rule
carbonTable);
case Alias(attr: AggregateExpression, _) =>
if (attr.isDistinct) {
- return false
+ isValid = false
}
val aggExp = PreAggregateUtil.validateAggregateFunctionAndGetFields(attr)
if (aggExp.nonEmpty) {
aggreagteExps ++= aggExp
} else {
- return false
+ isValid = false
}
case Alias(expression: Expression, _) =>
if (expression.isInstanceOf[ScalaUDF] &&
@@ -1049,10 +1050,22 @@ case class CarbonPreAggregateQueryRules(sparkSession: SparkSession) extends Rule
queryColumns += getQueryColumn(attr.name,
carbonTable)
attr
+ case attr: AggregateExpression =>
+ if (attr.isDistinct) {
+ isValid = false
+ }
+ val aggExp = PreAggregateUtil.validateAggregateFunctionAndGetFields(attr)
+ if (aggExp.nonEmpty) {
+ aggreagteExps ++= aggExp
+ } else {
+ isValid = false
+ }
+ attr
+
}
}
}
- true
+ isValid
}
/**