You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by hv...@apache.org on 2017/09/04 21:03:03 UTC
spark git commit: [SPARK-21418][SQL] NoSuchElementException: None.get
in DataSourceScanExec with sun.io.serialization.extendedDebugInfo=true
Repository: spark
Updated Branches:
refs/heads/master 9f30d9280 -> ca59445ad
[SPARK-21418][SQL] NoSuchElementException: None.get in DataSourceScanExec with sun.io.serialization.extendedDebugInfo=true
## What changes were proposed in this pull request?
If no SparkConf is available to Utils.redact, simply don't redact.
## How was this patch tested?
Existing tests
Author: Sean Owen <so...@cloudera.com>
Closes #19123 from srowen/SPARK-21418.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ca59445a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ca59445a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ca59445a
Branch: refs/heads/master
Commit: ca59445adb30ed796189532df2a2898ecd33db68
Parents: 9f30d92
Author: Sean Owen <so...@cloudera.com>
Authored: Mon Sep 4 23:02:59 2017 +0200
Committer: Herman van Hovell <hv...@databricks.com>
Committed: Mon Sep 4 23:02:59 2017 +0200
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/util/Utils.scala | 9 ++++++---
.../org/apache/spark/sql/execution/DataSourceScanExec.scala | 2 +-
2 files changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/ca59445a/core/src/main/scala/org/apache/spark/util/Utils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
index 0da075d..1e8250f 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -2639,9 +2639,12 @@ private[spark] object Utils extends Logging {
* Redact the sensitive information in the given string.
*/
def redact(conf: SparkConf, text: String): String = {
- if (text == null || text.isEmpty || !conf.contains(STRING_REDACTION_PATTERN)) return text
- val regex = conf.get(STRING_REDACTION_PATTERN).get
- regex.replaceAllIn(text, REDACTION_REPLACEMENT_TEXT)
+ if (text == null || text.isEmpty || conf == null || !conf.contains(STRING_REDACTION_PATTERN)) {
+ text
+ } else {
+ val regex = conf.get(STRING_REDACTION_PATTERN).get
+ regex.replaceAllIn(text, REDACTION_REPLACEMENT_TEXT)
+ }
}
private def redact(redactionPattern: Regex, kvs: Seq[(String, String)]): Seq[(String, String)] = {
http://git-wip-us.apache.org/repos/asf/spark/blob/ca59445a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
index 77e6dbf..8d0fc32 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala
@@ -69,7 +69,7 @@ trait DataSourceScanExec extends LeafExecNode with CodegenSupport {
* Shorthand for calling redactString() without specifying redacting rules
*/
private def redact(text: String): String = {
- Utils.redact(SparkSession.getActiveSession.get.sparkContext.conf, text)
+ Utils.redact(SparkSession.getActiveSession.map(_.sparkContext.conf).orNull, text)
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org