You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by zs...@apache.org on 2018/01/04 19:39:47 UTC
spark git commit: [SPARK-21475][CORE][2ND ATTEMPT] Change to use
NIO's Files API for external shuffle service
Repository: spark
Updated Branches:
refs/heads/master 6f68316e9 -> 93f92c0ed
[SPARK-21475][CORE][2ND ATTEMPT] Change to use NIO's Files API for external shuffle service
## What changes were proposed in this pull request?
This PR is the second attempt of #18684 , NIO's Files API doesn't override `skip` method for `InputStream`, so it will bring in performance issue (mentioned in #20119). But using `FileInputStream`/`FileOutputStream` will also bring in memory issue (https://dzone.com/articles/fileinputstream-fileoutputstream-considered-harmful), which is severe for long running external shuffle service. So here in this proposal, only fixing the external shuffle service related code.
## How was this patch tested?
Existing tests.
Author: jerryshao <ss...@hortonworks.com>
Closes #20144 from jerryshao/SPARK-21475-v2.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/93f92c0e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/93f92c0e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/93f92c0e
Branch: refs/heads/master
Commit: 93f92c0ed7442a4382e97254307309977ff676f8
Parents: 6f68316
Author: jerryshao <ss...@hortonworks.com>
Authored: Thu Jan 4 11:39:42 2018 -0800
Committer: Shixiong Zhu <zs...@gmail.com>
Committed: Thu Jan 4 11:39:42 2018 -0800
----------------------------------------------------------------------
.../apache/spark/network/buffer/FileSegmentManagedBuffer.java | 3 ++-
.../apache/spark/network/shuffle/ShuffleIndexInformation.java | 4 ++--
2 files changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/93f92c0e/common/network-common/src/main/java/org/apache/spark/network/buffer/FileSegmentManagedBuffer.java
----------------------------------------------------------------------
diff --git a/common/network-common/src/main/java/org/apache/spark/network/buffer/FileSegmentManagedBuffer.java b/common/network-common/src/main/java/org/apache/spark/network/buffer/FileSegmentManagedBuffer.java
index c20fab8..8b8f989 100644
--- a/common/network-common/src/main/java/org/apache/spark/network/buffer/FileSegmentManagedBuffer.java
+++ b/common/network-common/src/main/java/org/apache/spark/network/buffer/FileSegmentManagedBuffer.java
@@ -24,6 +24,7 @@ import java.io.InputStream;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
+import java.nio.file.StandardOpenOption;
import com.google.common.base.Objects;
import com.google.common.io.ByteStreams;
@@ -132,7 +133,7 @@ public final class FileSegmentManagedBuffer extends ManagedBuffer {
if (conf.lazyFileDescriptor()) {
return new DefaultFileRegion(file, offset, length);
} else {
- FileChannel fileChannel = new FileInputStream(file).getChannel();
+ FileChannel fileChannel = FileChannel.open(file.toPath(), StandardOpenOption.READ);
return new DefaultFileRegion(fileChannel, offset, length);
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/93f92c0e/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ShuffleIndexInformation.java
----------------------------------------------------------------------
diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ShuffleIndexInformation.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ShuffleIndexInformation.java
index eacf485..386738e 100644
--- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ShuffleIndexInformation.java
+++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/ShuffleIndexInformation.java
@@ -19,10 +19,10 @@ package org.apache.spark.network.shuffle;
import java.io.DataInputStream;
import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.LongBuffer;
+import java.nio.file.Files;
/**
* Keeps the index information for a particular map output
@@ -39,7 +39,7 @@ public class ShuffleIndexInformation {
offsets = buffer.asLongBuffer();
DataInputStream dis = null;
try {
- dis = new DataInputStream(new FileInputStream(indexFile));
+ dis = new DataInputStream(Files.newInputStream(indexFile.toPath()));
dis.readFully(buffer.array());
} finally {
if (dis != null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org