You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by li...@apache.org on 2017/06/05 23:01:05 UTC
hadoop git commit: HADOOP-14478. Optimize NativeAzureFsInputStream
for positional reads. Contributed by Rajesh Balamohan
Repository: hadoop
Updated Branches:
refs/heads/trunk 835560983 -> 5fd9742c8
HADOOP-14478. Optimize NativeAzureFsInputStream for positional reads. Contributed by Rajesh Balamohan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5fd9742c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5fd9742c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5fd9742c
Branch: refs/heads/trunk
Commit: 5fd9742c83fbeae96bf0913bdcdf77fafbf15b2f
Parents: 8355609
Author: Mingliang Liu <li...@apache.org>
Authored: Mon Jun 5 15:56:43 2017 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Mon Jun 5 15:56:43 2017 -0700
----------------------------------------------------------------------
.../fs/azure/AzureNativeFileSystemStore.java | 5 +---
.../hadoop/fs/azure/NativeAzureFileSystem.java | 31 ++++++++++++++++++--
2 files changed, 29 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fd9742c/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index a5bb370..534919e 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.fs.azure;
import static org.apache.hadoop.fs.azure.NativeAzureFileSystem.PATH_DELIMITER;
-import java.io.BufferedInputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
@@ -2043,11 +2042,9 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
// Get blob reference and open the input buffer stream.
CloudBlobWrapper blob = getBlobReference(key);
- BufferedInputStream inBufStream = new BufferedInputStream(
- openInputStream(blob));
// Return a data input stream.
- DataInputStream inDataStream = new DataInputStream(inBufStream);
+ DataInputStream inDataStream = new DataInputStream(openInputStream(blob));
return inDataStream;
} catch (Exception e) {
// Re-throw as an Azure storage exception.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fd9742c/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index b61baab..b2cc4ea 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -816,6 +816,27 @@ public class NativeAzureFileSystem extends FileSystem {
}
}
+ @Override
+ public synchronized void readFully(long position, byte[] buffer, int offset, int length)
+ throws IOException {
+ validatePositionedReadArgs(position, buffer, offset, length);
+
+ int nread = 0;
+ while (nread < length) {
+ // In case BlobInputStream is used, mark() can act as a hint to read ahead only this
+ // length instead of 4 MB boundary.
+ in.mark(length - nread);
+ int nbytes = read(position + nread,
+ buffer,
+ offset + nread,
+ length - nread);
+ if (nbytes < 0) {
+ throw new EOFException(FSExceptionMessages.EOF_IN_READ_FULLY);
+ }
+ nread += nbytes;
+ }
+ }
+
/*
* Reads up to len bytes of data from the input stream into an array of
* bytes. An attempt is made to read as many as len bytes, but a smaller
@@ -886,9 +907,13 @@ public class NativeAzureFileSystem extends FileSystem {
if (pos < 0) {
throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK);
}
- IOUtils.closeStream(in);
- in = store.retrieve(key);
- this.pos = in.skip(pos);
+ if (this.pos > pos) {
+ IOUtils.closeStream(in);
+ in = store.retrieve(key);
+ this.pos = in.skip(pos);
+ } else {
+ this.pos += in.skip(pos - this.pos);
+ }
LOG.debug("Seek to position {}. Bytes skipped {}", pos,
this.pos);
} catch(IOException e) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org