You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2015/04/03 03:30:59 UTC
spark git commit: [SPARK-6686][SQL] Use resolved output instead of
names for toDF rename
Repository: spark
Updated Branches:
refs/heads/master 947802cb0 -> 052dee070
[SPARK-6686][SQL] Use resolved output instead of names for toDF rename
This is a workaround for a problem reported on the user list. This doesn't fix the core problem, but in general is a more robust way to do renames.
Author: Michael Armbrust <mi...@databricks.com>
Closes #5337 from marmbrus/toDFrename and squashes the following commits:
6a3159d [Michael Armbrust] [SPARK-6686][SQL] Use resolved output instead of names for toDF rename
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/052dee07
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/052dee07
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/052dee07
Branch: refs/heads/master
Commit: 052dee0707830cfd3cd8821ecc3471a37ede294a
Parents: 947802c
Author: Michael Armbrust <mi...@databricks.com>
Authored: Thu Apr 2 18:30:55 2015 -0700
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Thu Apr 2 18:30:55 2015 -0700
----------------------------------------------------------------------
sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala | 4 ++--
.../src/test/scala/org/apache/spark/sql/DataFrameSuite.scala | 8 ++++++++
2 files changed, 10 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/052dee07/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
index 34be173..5c6016a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
@@ -240,8 +240,8 @@ class DataFrame private[sql](
s"Old column names (${schema.size}): " + schema.fields.map(_.name).mkString(", ") + "\n" +
s"New column names (${colNames.size}): " + colNames.mkString(", "))
- val newCols = schema.fieldNames.zip(colNames).map { case (oldName, newName) =>
- apply(oldName).as(newName)
+ val newCols = logicalPlan.output.zip(colNames).map { case (oldAttribute, newName) =>
+ Column(oldAttribute).as(newName)
}
select(newCols :_*)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/052dee07/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index 5297cc0..1db0cf7 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -60,6 +60,14 @@ class DataFrameSuite extends QueryTest {
assert($"test".toString === "test")
}
+ test("rename nested groupby") {
+ val df = Seq((1,(1,1))).toDF()
+
+ checkAnswer(
+ df.groupBy("_1").agg(col("_1"), sum("_2._1")).toDF("key", "total"),
+ Row(1, 1) :: Nil)
+ }
+
test("invalid plan toString, debug mode") {
val oldSetting = TestSQLContext.conf.dataFrameEagerAnalysis
TestSQLContext.setConf(SQLConf.DATAFRAME_EAGER_ANALYSIS, "true")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org