You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2014/10/21 00:32:35 UTC
git commit: [SPARK-3800][SQL] Clean aliases from grouping expressions
Repository: spark
Updated Branches:
refs/heads/master 1b3ce61ce -> e9c1afa87
[SPARK-3800][SQL] Clean aliases from grouping expressions
Author: Michael Armbrust <mi...@databricks.com>
Closes #2658 from marmbrus/nestedAggs and squashes the following commits:
862b763 [Michael Armbrust] Merge remote-tracking branch 'origin/master' into nestedAggs
3234521 [Michael Armbrust] Merge remote-tracking branch 'origin/master' into nestedAggs
8b06fdc [Michael Armbrust] possible fix for grouping on nested fields
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e9c1afa8
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e9c1afa8
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e9c1afa8
Branch: refs/heads/master
Commit: e9c1afa87b8a6d9ea02aa9f7c319a36f00c0e4c3
Parents: 1b3ce61
Author: Michael Armbrust <mi...@databricks.com>
Authored: Mon Oct 20 15:32:17 2014 -0700
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Mon Oct 20 15:32:17 2014 -0700
----------------------------------------------------------------------
.../spark/sql/catalyst/analysis/Analyzer.scala | 18 ++++++++++++++++++
.../org/apache/spark/sql/SQLQuerySuite.scala | 17 +++++++++++++++++
2 files changed, 35 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/e9c1afa8/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 8255306..a448c79 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -60,6 +60,7 @@ class Analyzer(catalog: Catalog, registry: FunctionRegistry, caseSensitive: Bool
ResolveFunctions ::
GlobalAggregates ::
UnresolvedHavingClauseAttributes ::
+ TrimAliases ::
typeCoercionRules ++
extendedRules : _*),
Batch("Check Analysis", Once,
@@ -90,6 +91,23 @@ class Analyzer(catalog: Catalog, registry: FunctionRegistry, caseSensitive: Bool
}
/**
+ * Removes no-op Alias expressions from the plan.
+ */
+ object TrimAliases extends Rule[LogicalPlan] {
+ def apply(plan: LogicalPlan): LogicalPlan = plan transform {
+ case Aggregate(groups, aggs, child) =>
+ Aggregate(
+ groups.map {
+ _ transform {
+ case Alias(c, _) => c
+ }
+ },
+ aggs,
+ child)
+ }
+ }
+
+ /**
* Checks for non-aggregated attributes with aggregation
*/
object CheckAggregation extends Rule[LogicalPlan] {
http://git-wip-us.apache.org/repos/asf/spark/blob/e9c1afa8/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index beb924f..3959925 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -43,6 +43,23 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll {
TimeZone.setDefault(origZone)
}
+ test("grouping on nested fields") {
+ jsonRDD(sparkContext.parallelize("""{"nested": {"attribute": 1}, "value": 2}""" :: Nil))
+ .registerTempTable("rows")
+
+ checkAnswer(
+ sql(
+ """
+ |select attribute, sum(cnt)
+ |from (
+ | select nested.attribute, count(*) as cnt
+ | from rows
+ | group by nested.attribute) a
+ |group by attribute
+ """.stripMargin),
+ Row(1, 1) :: Nil)
+ }
+
test("SPARK-3176 Added Parser of SQL ABS()") {
checkAnswer(
sql("SELECT ABS(-1.3)"),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org