You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ir...@apache.org on 2017/12/20 17:31:23 UTC
spark git commit: [SPARK-22788][STREAMING] Use correct hadoop config
for fs append support.
Repository: spark
Updated Branches:
refs/heads/master 9962390af -> 7570eab6b
[SPARK-22788][STREAMING] Use correct hadoop config for fs append support.
Still look at the old one in case any Spark user is setting it
explicitly, though.
Author: Marcelo Vanzin <va...@cloudera.com>
Closes #19983 from vanzin/SPARK-22788.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/7570eab6
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/7570eab6
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/7570eab6
Branch: refs/heads/master
Commit: 7570eab6bee57172ee3746207261307690a57b72
Parents: 9962390
Author: Marcelo Vanzin <va...@cloudera.com>
Authored: Wed Dec 20 11:31:11 2017 -0600
Committer: Imran Rashid <ir...@cloudera.com>
Committed: Wed Dec 20 11:31:11 2017 -0600
----------------------------------------------------------------------
.../main/scala/org/apache/spark/streaming/util/HdfsUtils.scala | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/7570eab6/streaming/src/main/scala/org/apache/spark/streaming/util/HdfsUtils.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/spark/streaming/util/HdfsUtils.scala b/streaming/src/main/scala/org/apache/spark/streaming/util/HdfsUtils.scala
index 6a3b320..a699735 100644
--- a/streaming/src/main/scala/org/apache/spark/streaming/util/HdfsUtils.scala
+++ b/streaming/src/main/scala/org/apache/spark/streaming/util/HdfsUtils.scala
@@ -29,7 +29,9 @@ private[streaming] object HdfsUtils {
// If the file exists and we have append support, append instead of creating a new file
val stream: FSDataOutputStream = {
if (dfs.isFile(dfsPath)) {
- if (conf.getBoolean("hdfs.append.support", false) || dfs.isInstanceOf[RawLocalFileSystem]) {
+ if (conf.getBoolean("dfs.support.append", true) ||
+ conf.getBoolean("hdfs.append.support", false) ||
+ dfs.isInstanceOf[RawLocalFileSystem]) {
dfs.append(dfsPath)
} else {
throw new IllegalStateException("File exists and there is no append support!")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org