You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2016/03/18 16:16:25 UTC
spark git commit: [SPARK-13972][SQ] hive tests should fail if SQL
generation failed
Repository: spark
Updated Branches:
refs/heads/master 53f32a22d -> 0acb32a3f
[SPARK-13972][SQ] hive tests should fail if SQL generation failed
## What changes were proposed in this pull request?
Now we should be able to convert all logical plans to SQL string, if they are parsed from hive query. This PR changes the error handling to throw exceptions instead of just log.
We will send new PRs for spotted bugs, and merge this one after all bugs are fixed.
## How was this patch tested?
existing tests.
Author: Wenchen Fan <we...@databricks.com>
Closes #11782 from cloud-fan/test.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0acb32a3
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0acb32a3
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0acb32a3
Branch: refs/heads/master
Commit: 0acb32a3f19484d3ea3b60fccef596025c8a8f83
Parents: 53f32a2
Author: Wenchen Fan <we...@databricks.com>
Authored: Fri Mar 18 23:16:14 2016 +0800
Committer: Cheng Lian <li...@databricks.com>
Committed: Fri Mar 18 23:16:14 2016 +0800
----------------------------------------------------------------------
.../sql/hive/execution/HiveComparisonTest.scala | 71 ++++++++------------
1 file changed, 28 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/0acb32a3/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
index d21bb57..019d752 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveComparisonTest.scala
@@ -132,25 +132,7 @@ abstract class HiveComparisonTest
new java.math.BigInteger(1, digest.digest).toString(16)
}
- /** Used for testing [[SQLBuilder]] */
- private var numConvertibleQueries: Int = 0
- private var numTotalQueries: Int = 0
-
override protected def afterAll(): Unit = {
- logInfo({
- val percentage = if (numTotalQueries > 0) {
- numConvertibleQueries.toDouble / numTotalQueries * 100
- } else {
- 0D
- }
-
- s"""SQLBuilder statistics:
- |- Total query number: $numTotalQueries
- |- Number of convertible queries: $numConvertibleQueries
- |- Percentage of convertible queries: $percentage%
- """.stripMargin
- })
-
try {
TestHive.reset()
} finally {
@@ -412,32 +394,35 @@ abstract class HiveComparisonTest
if (containsCommands) {
originalQuery
} else {
- numTotalQueries += 1
+ val convertedSQL = try {
+ new SQLBuilder(originalQuery.analyzed, TestHive).toSQL
+ } catch {
+ case NonFatal(e) => fail(
+ s"""Cannot convert the following HiveQL query plan back to SQL query string:
+ |
+ |# Original HiveQL query string:
+ |$queryString
+ |
+ |# Resolved query plan:
+ |${originalQuery.analyzed.treeString}
+ """.stripMargin, e)
+ }
+
try {
- val sql = new SQLBuilder(originalQuery.analyzed, TestHive).toSQL
- numConvertibleQueries += 1
- logInfo(
- s"""
- |### Running SQL generation round-trip test {{{
- |${originalQuery.analyzed.treeString}
- |Original SQL:
- |$queryString
- |
- |Generated SQL:
- |$sql
- |}}}
- """.stripMargin.trim)
- new TestHive.QueryExecution(sql)
- } catch { case NonFatal(e) =>
- logInfo(
- s"""
- |### Cannot convert the following logical plan back to SQL {{{
- |${originalQuery.analyzed.treeString}
- |Original SQL:
- |$queryString
- |}}}
- """.stripMargin.trim)
- originalQuery
+ new TestHive.QueryExecution(convertedSQL)
+ } catch {
+ case NonFatal(e) => fail(
+ s"""Failed to analyze the converted SQL string:
+ |
+ |# Original HiveQL query string:
+ |$queryString
+ |
+ |# Resolved query plan:
+ |${originalQuery.analyzed.treeString}
+ |
+ |# Converted SQL query string:
+ |$convertedSQL
+ """.stripMargin, e)
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org