You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ir...@apache.org on 2017/12/20 17:31:23 UTC

spark git commit: [SPARK-22788][STREAMING] Use correct hadoop config for fs append support.

Repository: spark
Updated Branches:
  refs/heads/master 9962390af -> 7570eab6b


[SPARK-22788][STREAMING] Use correct hadoop config for fs append support.

Still look at the old one in case any Spark user is setting it
explicitly, though.

Author: Marcelo Vanzin <va...@cloudera.com>

Closes #19983 from vanzin/SPARK-22788.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/7570eab6
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/7570eab6
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/7570eab6

Branch: refs/heads/master
Commit: 7570eab6bee57172ee3746207261307690a57b72
Parents: 9962390
Author: Marcelo Vanzin <va...@cloudera.com>
Authored: Wed Dec 20 11:31:11 2017 -0600
Committer: Imran Rashid <ir...@cloudera.com>
Committed: Wed Dec 20 11:31:11 2017 -0600

----------------------------------------------------------------------
 .../main/scala/org/apache/spark/streaming/util/HdfsUtils.scala   | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/7570eab6/streaming/src/main/scala/org/apache/spark/streaming/util/HdfsUtils.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/util/HdfsUtils.scala b/streaming/src/main/scala/org/apache/spark/streaming/util/HdfsUtils.scala
index 6a3b320..a699735 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/util/HdfsUtils.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/util/HdfsUtils.scala
@@ -29,7 +29,9 @@ private[streaming] object HdfsUtils {
     // If the file exists and we have append support, append instead of creating a new file
     val stream: FSDataOutputStream = {
       if (dfs.isFile(dfsPath)) {
-        if (conf.getBoolean("hdfs.append.support", false) || dfs.isInstanceOf[RawLocalFileSystem]) {
+        if (conf.getBoolean("dfs.support.append", true) ||
+            conf.getBoolean("hdfs.append.support", false) ||
+            dfs.isInstanceOf[RawLocalFileSystem]) {
           dfs.append(dfsPath)
         } else {
           throw new IllegalStateException("File exists and there is no append support!")


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org