You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2014/07/22 03:18:27 UTC
git commit: [SPARK-2561][SQL] Fix apply schema
Repository: spark
Updated Branches:
refs/heads/master a4d60208e -> 511a73140
[SPARK-2561][SQL] Fix apply schema
We need to use the analyzed attributes otherwise we end up with a tree that will never resolve.
Author: Michael Armbrust <mi...@databricks.com>
Closes #1470 from marmbrus/fixApplySchema and squashes the following commits:
f968195 [Michael Armbrust] Use analyzed attributes when applying the schema.
4969015 [Michael Armbrust] Add test case.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/511a7314
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/511a7314
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/511a7314
Branch: refs/heads/master
Commit: 511a7314037219c23e824ea5363bf7f1df55bab3
Parents: a4d6020
Author: Michael Armbrust <mi...@databricks.com>
Authored: Mon Jul 21 18:18:17 2014 -0700
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Mon Jul 21 18:18:17 2014 -0700
----------------------------------------------------------------------
sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala | 2 +-
.../src/test/scala/org/apache/spark/sql/DslQuerySuite.scala | 6 ++++++
2 files changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/511a7314/sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala
index 993d085..31d27bb 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala
@@ -430,7 +430,7 @@ class SchemaRDD(
* @group schema
*/
private def applySchema(rdd: RDD[Row]): SchemaRDD = {
- new SchemaRDD(sqlContext, SparkLogicalPlan(ExistingRdd(logicalPlan.output, rdd)))
+ new SchemaRDD(sqlContext, SparkLogicalPlan(ExistingRdd(queryExecution.analyzed.output, rdd)))
}
// =======================================================================
http://git-wip-us.apache.org/repos/asf/spark/blob/511a7314/sql/core/src/test/scala/org/apache/spark/sql/DslQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DslQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DslQuerySuite.scala
index 68dae58..c8ea01c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DslQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DslQuerySuite.scala
@@ -33,6 +33,12 @@ class DslQuerySuite extends QueryTest {
testData.collect().toSeq)
}
+ test("repartition") {
+ checkAnswer(
+ testData.select('key).repartition(10).select('key),
+ testData.select('key).collect().toSeq)
+ }
+
test("agg") {
checkAnswer(
testData2.groupBy('a)('a, Sum('b)),