You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2015/01/30 03:23:12 UTC
spark git commit: [SPARK-5462] [SQL] Use analyzed query plan in
DataFrame.apply()
Repository: spark
Updated Branches:
refs/heads/master 5c746eedd -> 22271f969
[SPARK-5462] [SQL] Use analyzed query plan in DataFrame.apply()
This patch changes DataFrame's `apply()` method to use an analyzed query plan when resolving column names. This fixes a bug where `apply` would throw "invalid call to qualifiers on unresolved object" errors when called on DataFrames constructed via `SQLContext.sql()`.
Author: Josh Rosen <jo...@databricks.com>
Closes #4282 from JoshRosen/SPARK-5462 and squashes the following commits:
b9e6da2 [Josh Rosen] [SPARK-5462] Use analyzed query plan in DataFrame.apply().
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/22271f96
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/22271f96
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/22271f96
Branch: refs/heads/master
Commit: 22271f969363fd139e6cfb5a2d95a2607fb4e572
Parents: 5c746ee
Author: Josh Rosen <jo...@databricks.com>
Authored: Thu Jan 29 18:23:05 2015 -0800
Committer: Reynold Xin <rx...@databricks.com>
Committed: Thu Jan 29 18:23:05 2015 -0800
----------------------------------------------------------------------
sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala | 8 +++++---
.../src/test/scala/org/apache/spark/sql/DataFrameSuite.scala | 4 ++++
2 files changed, 9 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/22271f96/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
index 1ff25ad..2694e81 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
@@ -111,14 +111,16 @@ class DataFrame protected[sql](
/** Returns the list of numeric columns, useful for doing aggregation. */
protected[sql] def numericColumns: Seq[Expression] = {
schema.fields.filter(_.dataType.isInstanceOf[NumericType]).map { n =>
- logicalPlan.resolve(n.name, sqlContext.analyzer.resolver).get
+ queryExecution.analyzed.resolve(n.name, sqlContext.analyzer.resolver).get
}
}
/** Resolves a column name into a Catalyst [[NamedExpression]]. */
protected[sql] def resolve(colName: String): NamedExpression = {
- logicalPlan.resolve(colName, sqlContext.analyzer.resolver).getOrElse(throw new RuntimeException(
- s"""Cannot resolve column name "$colName" among (${schema.fieldNames.mkString(", ")})"""))
+ queryExecution.analyzed.resolve(colName, sqlContext.analyzer.resolver).getOrElse {
+ throw new RuntimeException(
+ s"""Cannot resolve column name "$colName" among (${schema.fieldNames.mkString(", ")})""")
+ }
}
/** Left here for compatibility reasons. */
http://git-wip-us.apache.org/repos/asf/spark/blob/22271f96/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index db83a90..df343ad 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -276,5 +276,9 @@ class DataFrameSuite extends QueryTest {
)
}
+ test("apply on query results (SPARK-5462)") {
+ val df = testData.sqlContext.sql("select key from testData")
+ checkAnswer(df("key"), testData.select('key).collect().toSeq)
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org