You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by zs...@apache.org on 2017/01/10 18:49:46 UTC
spark git commit: [SPARK-19137][SQL] Fix `withSQLConf` to reset
`OptionalConfigEntry` correctly
Repository: spark
Updated Branches:
refs/heads/master 3ef183a94 -> d5b1dc934
[SPARK-19137][SQL] Fix `withSQLConf` to reset `OptionalConfigEntry` correctly
## What changes were proposed in this pull request?
`DataStreamReaderWriterSuite` makes test files in source folder like the followings. Interestingly, the root cause is `withSQLConf` fails to reset `OptionalConfigEntry` correctly. In other words, it resets the config into `Some(undefined)`.
```bash
$ git status
Untracked files:
(use "git add <file>..." to include in what will be committed)
sql/core/%253Cundefined%253E/
sql/core/%3Cundefined%3E/
```
## How was this patch tested?
Manual.
```
build/sbt "project sql" test
git status
```
Author: Dongjoon Hyun <do...@apache.org>
Closes #16522 from dongjoon-hyun/SPARK-19137.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d5b1dc93
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d5b1dc93
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d5b1dc93
Branch: refs/heads/master
Commit: d5b1dc934a2482886c2c095de90e4c6a49ec42bd
Parents: 3ef183a
Author: Dongjoon Hyun <do...@apache.org>
Authored: Tue Jan 10 10:49:44 2017 -0800
Committer: Shixiong Zhu <sh...@databricks.com>
Committed: Tue Jan 10 10:49:44 2017 -0800
----------------------------------------------------------------------
.../test/scala/org/apache/spark/sql/test/SQLTestUtils.scala | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/d5b1dc93/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
index d4d8e3e..d4afb9d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/test/SQLTestUtils.scala
@@ -94,7 +94,13 @@ private[sql] trait SQLTestUtils
*/
protected def withSQLConf(pairs: (String, String)*)(f: => Unit): Unit = {
val (keys, values) = pairs.unzip
- val currentValues = keys.map(key => Try(spark.conf.get(key)).toOption)
+ val currentValues = keys.map { key =>
+ if (spark.conf.contains(key)) {
+ Some(spark.conf.get(key))
+ } else {
+ None
+ }
+ }
(keys, values).zipped.foreach(spark.conf.set)
try f finally {
keys.zip(currentValues).foreach {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org