You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2015/07/22 08:00:21 UTC
spark git commit: [SPARK-9232] [SQL] Duplicate code in JSONRelation
Repository: spark
Updated Branches:
refs/heads/master 63f4bcc73 -> f4785f5b8
[SPARK-9232] [SQL] Duplicate code in JSONRelation
Author: Andrew Or <an...@databricks.com>
Closes #7576 from andrewor14/clean-up-json-relation and squashes the following commits:
ea80803 [Andrew Or] Clean up duplicate code
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f4785f5b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f4785f5b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f4785f5b
Branch: refs/heads/master
Commit: f4785f5b82c57bce41d3dc26ed9e3c9e794c7558
Parents: 63f4bcc
Author: Andrew Or <an...@databricks.com>
Authored: Tue Jul 21 23:00:13 2015 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Tue Jul 21 23:00:13 2015 -0700
----------------------------------------------------------------------
.../apache/spark/sql/json/JSONRelation.scala | 50 ++++++++------------
1 file changed, 21 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/f4785f5b/sql/core/src/main/scala/org/apache/spark/sql/json/JSONRelation.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/json/JSONRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/json/JSONRelation.scala
index 25802d0..922794a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/json/JSONRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/json/JSONRelation.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.json
import java.io.IOException
-import org.apache.hadoop.fs.Path
+import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.AnalysisException
@@ -87,20 +87,7 @@ private[sql] class DefaultSource
case SaveMode.Append =>
sys.error(s"Append mode is not supported by ${this.getClass.getCanonicalName}")
case SaveMode.Overwrite => {
- var success: Boolean = false
- try {
- success = fs.delete(filesystemPath, true)
- } catch {
- case e: IOException =>
- throw new IOException(
- s"Unable to clear output directory ${filesystemPath.toString} prior"
- + s" to writing to JSON table:\n${e.toString}")
- }
- if (!success) {
- throw new IOException(
- s"Unable to clear output directory ${filesystemPath.toString} prior"
- + s" to writing to JSON table.")
- }
+ JSONRelation.delete(filesystemPath, fs)
true
}
case SaveMode.ErrorIfExists =>
@@ -195,20 +182,7 @@ private[sql] class JSONRelation(
if (overwrite) {
if (fs.exists(filesystemPath)) {
- var success: Boolean = false
- try {
- success = fs.delete(filesystemPath, true)
- } catch {
- case e: IOException =>
- throw new IOException(
- s"Unable to clear output directory ${filesystemPath.toString} prior"
- + s" to writing to JSON table:\n${e.toString}")
- }
- if (!success) {
- throw new IOException(
- s"Unable to clear output directory ${filesystemPath.toString} prior"
- + s" to writing to JSON table.")
- }
+ JSONRelation.delete(filesystemPath, fs)
}
// Write the data.
data.toJSON.saveAsTextFile(filesystemPath.toString)
@@ -228,3 +202,21 @@ private[sql] class JSONRelation(
case _ => false
}
}
+
+private object JSONRelation {
+
+ /** Delete the specified directory to overwrite it with new JSON data. */
+ def delete(dir: Path, fs: FileSystem): Unit = {
+ var success: Boolean = false
+ val failMessage = s"Unable to clear output directory $dir prior to writing to JSON table"
+ try {
+ success = fs.delete(dir, true /* recursive */)
+ } catch {
+ case e: IOException =>
+ throw new IOException(s"$failMessage\n${e.toString}")
+ }
+ if (!success) {
+ throw new IOException(failMessage)
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org