You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2016/05/09 18:14:58 UTC
spark git commit: [MINOR][SQL] Enhance the exception message if
checkpointLocation is not set
Repository: spark
Updated Branches:
refs/heads/master 6747171eb -> ee6a8d7ea
[MINOR][SQL] Enhance the exception message if checkpointLocation is not set
Enhance the exception message when `checkpointLocation` is not set, previously the message is:
```
java.util.NoSuchElementException: None.get
at scala.None$.get(Option.scala:347)
at scala.None$.get(Option.scala:345)
at org.apache.spark.sql.DataFrameWriter$$anonfun$8.apply(DataFrameWriter.scala:338)
at org.apache.spark.sql.DataFrameWriter$$anonfun$8.apply(DataFrameWriter.scala:338)
at scala.collection.MapLike$class.getOrElse(MapLike.scala:128)
at scala.collection.AbstractMap.getOrElse(Map.scala:59)
at org.apache.spark.sql.DataFrameWriter.startStream(DataFrameWriter.scala:337)
at org.apache.spark.sql.DataFrameWriter.startStream(DataFrameWriter.scala:277)
... 48 elided
```
This is not so meaningful, so changing to make it more specific.
Local verified.
Author: jerryshao <ss...@hortonworks.com>
Closes #12998 from jerryshao/improve-exception-message.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ee6a8d7e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ee6a8d7e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ee6a8d7e
Branch: refs/heads/master
Commit: ee6a8d7eaaafafb7eee6f9f2d633759b12732062
Parents: 6747171
Author: jerryshao <ss...@hortonworks.com>
Authored: Mon May 9 11:13:29 2016 -0700
Committer: Andrew Or <an...@databricks.com>
Committed: Mon May 9 11:14:40 2016 -0700
----------------------------------------------------------------------
.../scala/org/apache/spark/sql/DataFrameWriter.scala | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/ee6a8d7e/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
index 0793b62..6b1ccbe 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
@@ -334,9 +334,15 @@ final class DataFrameWriter private[sql](df: DataFrame) {
partitionColumns = normalizedParCols.getOrElse(Nil))
val queryName = extraOptions.getOrElse("queryName", StreamExecution.nextName)
- val checkpointLocation = extraOptions.getOrElse("checkpointLocation",
- new Path(df.sparkSession.sessionState.conf.checkpointLocation.get, queryName).toUri.toString
- )
+ val checkpointLocation = extraOptions.get("checkpointLocation")
+ .orElse {
+ df.sparkSession.sessionState.conf.checkpointLocation.map { l =>
+ new Path(l, queryName).toUri.toString
+ }
+ }.getOrElse {
+ throw new AnalysisException("checkpointLocation must be specified either " +
+ "through option() or SQLConf")
+ }
df.sparkSession.sessionState.continuousQueryManager.startQuery(
queryName,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org