You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2016/03/25 09:17:26 UTC
spark git commit: [SPARK-14149] Log exceptions in tryOrIOException
Repository: spark
Updated Branches:
refs/heads/master 20ddf5fdd -> 70a6f0bb5
[SPARK-14149] Log exceptions in tryOrIOException
## What changes were proposed in this pull request?
We ran into a problem today debugging some class loading problem during deserialization, and JVM was masking the underlying exception which made it very difficult to debug. We can however log the exceptions using try/catch ourselves in serialization/deserialization. The good thing is that all these methods are already using Utils.tryOrIOException, so we can just put the try catch and logging in a single place.
## How was this patch tested?
A logging change with a manual test.
Author: Reynold Xin <rx...@databricks.com>
Closes #11951 from rxin/SPARK-14149.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/70a6f0bb
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/70a6f0bb
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/70a6f0bb
Branch: refs/heads/master
Commit: 70a6f0bb57ca2248444157e2707fbcc3cb04e3bc
Parents: 20ddf5f
Author: Reynold Xin <rx...@databricks.com>
Authored: Fri Mar 25 01:17:23 2016 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Fri Mar 25 01:17:23 2016 -0700
----------------------------------------------------------------------
.../scala/org/apache/spark/util/Utils.scala | 23 +++++---------------
.../InsertIntoHadoopFsRelation.scala | 8 +++----
2 files changed, 9 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/70a6f0bb/core/src/main/scala/org/apache/spark/util/Utils.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
index fe6063c..73768ff 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -1194,21 +1194,6 @@ private[spark] object Utils extends Logging {
}
/**
- * Execute a block of code that evaluates to Unit, re-throwing any non-fatal uncaught
- * exceptions as IOException. This is used when implementing Externalizable and Serializable's
- * read and write methods, since Java's serializer will not report non-IOExceptions properly;
- * see SPARK-4080 for more context.
- */
- def tryOrIOException(block: => Unit) {
- try {
- block
- } catch {
- case e: IOException => throw e
- case NonFatal(t) => throw new IOException(t)
- }
- }
-
- /**
* Execute a block of code that returns a value, re-throwing any non-fatal uncaught
* exceptions as IOException. This is used when implementing Externalizable and Serializable's
* read and write methods, since Java's serializer will not report non-IOExceptions properly;
@@ -1218,8 +1203,12 @@ private[spark] object Utils extends Logging {
try {
block
} catch {
- case e: IOException => throw e
- case NonFatal(t) => throw new IOException(t)
+ case e: IOException =>
+ logError("Exception encountered", e)
+ throw e
+ case NonFatal(e) =>
+ logError("Exception encountered", e)
+ throw new IOException(e)
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/70a6f0bb/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelation.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelation.scala
index a30b520..e31380e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InsertIntoHadoopFsRelation.scala
@@ -88,11 +88,9 @@ private[sql] case class InsertIntoHadoopFsRelation(
case (SaveMode.ErrorIfExists, true) =>
throw new AnalysisException(s"path $qualifiedOutputPath already exists.")
case (SaveMode.Overwrite, true) =>
- Utils.tryOrIOException {
- if (!fs.delete(qualifiedOutputPath, true /* recursively */)) {
- throw new IOException(s"Unable to clear output " +
- s"directory $qualifiedOutputPath prior to writing to it")
- }
+ if (!fs.delete(qualifiedOutputPath, true /* recursively */)) {
+ throw new IOException(s"Unable to clear output " +
+ s"directory $qualifiedOutputPath prior to writing to it")
}
true
case (SaveMode.Append, _) | (SaveMode.Overwrite, _) | (SaveMode.ErrorIfExists, false) =>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org