You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by yh...@apache.org on 2015/08/09 20:44:58 UTC
spark git commit: [SPARK-8930] [SQL] Throw a AnalysisException with
meaningful messages if DataFrame#explode takes a star in expressions
Repository: spark
Updated Branches:
refs/heads/master e9c36938b -> 68ccc6e18
[SPARK-8930] [SQL] Throw a AnalysisException with meaningful messages if DataFrame#explode takes a star in expressions
Author: Yijie Shen <he...@gmail.com>
Closes #8057 from yjshen/explode_star and squashes the following commits:
eae181d [Yijie Shen] change explaination message
54c9d11 [Yijie Shen] meaning message for * in explode
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/68ccc6e1
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/68ccc6e1
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/68ccc6e1
Branch: refs/heads/master
Commit: 68ccc6e184598822b19a880fdd4597b66a1c2d92
Parents: e9c3693
Author: Yijie Shen <he...@gmail.com>
Authored: Sun Aug 9 11:44:51 2015 -0700
Committer: Yin Huai <yh...@databricks.com>
Committed: Sun Aug 9 11:44:51 2015 -0700
----------------------------------------------------------------------
.../spark/sql/catalyst/analysis/Analyzer.scala | 4 +++-
.../spark/sql/catalyst/analysis/AnalysisTest.scala | 4 +++-
.../scala/org/apache/spark/sql/DataFrameSuite.scala | 15 +++++++++++++++
3 files changed, 21 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/68ccc6e1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 82158e6..a684dbc 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -408,7 +408,7 @@ class Analyzer(
/**
* Returns true if `exprs` contains a [[Star]].
*/
- protected def containsStar(exprs: Seq[Expression]): Boolean =
+ def containsStar(exprs: Seq[Expression]): Boolean =
exprs.exists(_.collect { case _: Star => true }.nonEmpty)
}
@@ -602,6 +602,8 @@ class Analyzer(
*/
object ResolveGenerate extends Rule[LogicalPlan] {
def apply(plan: LogicalPlan): LogicalPlan = plan resolveOperators {
+ case g: Generate if ResolveReferences.containsStar(g.generator.children) =>
+ failAnalysis("Cannot explode *, explode can only be applied on a specific column.")
case p: Generate if !p.child.resolved || !p.generator.resolved => p
case g: Generate if !g.resolved =>
g.copy(generatorOutput = makeGeneratorOutput(g.generator, g.generatorOutput.map(_.name)))
http://git-wip-us.apache.org/repos/asf/spark/blob/68ccc6e1/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala
index ee1f8f5..53b3695 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala
@@ -71,6 +71,8 @@ trait AnalysisTest extends PlanTest {
val e = intercept[Exception] {
analyzer.checkAnalysis(analyzer.execute(inputPlan))
}
- expectedErrors.forall(e.getMessage.contains)
+ assert(expectedErrors.map(_.toLowerCase).forall(e.getMessage.toLowerCase.contains),
+ s"Expected to throw Exception contains: ${expectedErrors.mkString(", ")}, " +
+ s"actually we get ${e.getMessage}")
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/68ccc6e1/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index 0212637..c49f256 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -134,6 +134,21 @@ class DataFrameSuite extends QueryTest with SQLTestUtils {
)
}
+ test("SPARK-8930: explode should fail with a meaningful message if it takes a star") {
+ val df = Seq(("1", "1,2"), ("2", "4"), ("3", "7,8,9")).toDF("prefix", "csv")
+ val e = intercept[AnalysisException] {
+ df.explode($"*") { case Row(prefix: String, csv: String) =>
+ csv.split(",").map(v => Tuple1(prefix + ":" + v)).toSeq
+ }.queryExecution.assertAnalyzed()
+ }
+ assert(e.getMessage.contains(
+ "Cannot explode *, explode can only be applied on a specific column."))
+
+ df.explode('prefix, 'csv) { case Row(prefix: String, csv: String) =>
+ csv.split(",").map(v => Tuple1(prefix + ":" + v)).toSeq
+ }.queryExecution.assertAnalyzed()
+ }
+
test("explode alias and star") {
val df = Seq((Array("a"), 1)).toDF("a", "b")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org