You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by yh...@apache.org on 2015/12/08 19:52:23 UTC
spark git commit: [SPARK-12205][SQL] Pivot fails Analysis when
aggregate is UnresolvedFunction
Repository: spark
Updated Branches:
refs/heads/master 872a2ee28 -> 4bcb89494
[SPARK-12205][SQL] Pivot fails Analysis when aggregate is UnresolvedFunction
Delays application of ResolvePivot until all aggregates are resolved to prevent problems with UnresolvedFunction and adds unit test
Author: Andrew Ray <ra...@gmail.com>
Closes #10202 from aray/sql-pivot-unresolved-function.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4bcb8949
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4bcb8949
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4bcb8949
Branch: refs/heads/master
Commit: 4bcb894948c1b7294d84e2bf58abb1d79e6759c6
Parents: 872a2ee
Author: Andrew Ray <ra...@gmail.com>
Authored: Tue Dec 8 10:52:17 2015 -0800
Committer: Yin Huai <yh...@databricks.com>
Committed: Tue Dec 8 10:52:17 2015 -0800
----------------------------------------------------------------------
.../org/apache/spark/sql/catalyst/analysis/Analyzer.scala | 2 +-
.../scala/org/apache/spark/sql/DataFramePivotSuite.scala | 8 ++++++++
2 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/4bcb8949/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index d3163dc..ca00a5e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -259,7 +259,7 @@ class Analyzer(
object ResolvePivot extends Rule[LogicalPlan] {
def apply(plan: LogicalPlan): LogicalPlan = plan transform {
- case p: Pivot if !p.childrenResolved => p
+ case p: Pivot if !p.childrenResolved | !p.aggregates.forall(_.resolved) => p
case Pivot(groupByExprs, pivotColumn, pivotValues, aggregates, child) =>
val singleAgg = aggregates.size == 1
val pivotAggregates: Seq[NamedExpression] = pivotValues.flatMap { value =>
http://git-wip-us.apache.org/repos/asf/spark/blob/4bcb8949/sql/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala
index fc53aba..bc1a336 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala
@@ -85,4 +85,12 @@ class DataFramePivotSuite extends QueryTest with SharedSQLContext{
sqlContext.conf.setConf(SQLConf.DATAFRAME_PIVOT_MAX_VALUES,
SQLConf.DATAFRAME_PIVOT_MAX_VALUES.defaultValue.get)
}
+
+ test("pivot with UnresolvedFunction") {
+ checkAnswer(
+ courseSales.groupBy("year").pivot("course", Seq("dotNET", "Java"))
+ .agg("earnings" -> "sum"),
+ Row(2012, 15000.0, 20000.0) :: Row(2013, 48000.0, 30000.0) :: Nil
+ )
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org