You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2018/09/10 13:25:53 UTC
[2/2] hadoop git commit: HADOOP-15677. WASB: Add support for
StreamCapabilities. Contributed by Thomas Marquardt.
HADOOP-15677. WASB: Add support for StreamCapabilities.
Contributed by Thomas Marquardt.
(cherry picked from commit 1eef0acfe5725d08d3a4eb76c5138076ff1857f1)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1c2c0ed2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1c2c0ed2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1c2c0ed2
Branch: refs/heads/branch-3.1
Commit: 1c2c0ed2096a69ac4751e48703345fe460a41340
Parents: b8eaeca
Author: Steve Loughran <st...@apache.org>
Authored: Mon Sep 10 14:25:42 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Sep 10 14:25:42 2018 +0100
----------------------------------------------------------------------
.../hadoop/fs/azure/PageBlobOutputStream.java | 21 +++++++++-
.../fs/azure/ITestOutputStreamSemantics.java | 43 ++++++++++++++++++++
2 files changed, 63 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c2c0ed2/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java
index 68ddcdf..a1fe89d 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/PageBlobOutputStream.java
@@ -29,11 +29,13 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Arrays;
+import java.util.Locale;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.fs.StreamCapabilities;
import org.apache.hadoop.fs.Syncable;
import org.apache.hadoop.fs.azure.StorageInterface.CloudPageBlobWrapper;
import org.apache.commons.lang.exception.ExceptionUtils;
@@ -52,7 +54,7 @@ import com.microsoft.azure.storage.blob.CloudPageBlob;
* An output stream that write file data to a page blob stored using ASV's
* custom format.
*/
-final class PageBlobOutputStream extends OutputStream implements Syncable {
+final class PageBlobOutputStream extends OutputStream implements Syncable, StreamCapabilities {
/**
* The maximum number of raw bytes Azure Storage allows us to upload in a
* single request (4 MB).
@@ -196,6 +198,23 @@ final class PageBlobOutputStream extends OutputStream implements Syncable {
}
/**
+ * Query the stream for a specific capability.
+ *
+ * @param capability string to query the stream support for.
+ * @return true for hsync and hflush.
+ */
+ @Override
+ public boolean hasCapability(String capability) {
+ switch (capability.toLowerCase(Locale.ENGLISH)) {
+ case StreamCapabilities.HSYNC:
+ case StreamCapabilities.HFLUSH:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ /**
* Closes this output stream and releases any system resources associated with
* this stream. If any data remains in the buffer it is committed to the
* service.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c2c0ed2/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestOutputStreamSemantics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestOutputStreamSemantics.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestOutputStreamSemantics.java
index 9ac1f73..b8edc4b 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestOutputStreamSemantics.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/ITestOutputStreamSemantics.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StreamCapabilities;
import org.hamcrest.core.IsEqual;
import org.hamcrest.core.IsNot;
import org.junit.Test;
@@ -186,6 +187,20 @@ public class ITestOutputStreamSemantics extends AbstractWasbTestBase {
}
}
+ // Page Blobs have StreamCapabilities.HFLUSH and StreamCapabilities.HSYNC.
+ @Test
+ public void testPageBlobCapabilities() throws IOException {
+ Path path = getBlobPathWithTestName(PAGE_BLOB_DIR);
+ try (FSDataOutputStream stream = fs.create(path)) {
+ assertTrue(stream.hasCapability(StreamCapabilities.HFLUSH));
+ assertTrue(stream.hasCapability(StreamCapabilities.HSYNC));
+ assertFalse(stream.hasCapability(StreamCapabilities.DROPBEHIND));
+ assertFalse(stream.hasCapability(StreamCapabilities.READAHEAD));
+ assertFalse(stream.hasCapability(StreamCapabilities.UNBUFFER));
+ stream.write(getRandomBytes());
+ }
+ }
+
// Verify flush does not write data to storage for Block Blobs
@Test
public void testBlockBlobFlush() throws Exception {
@@ -265,6 +280,20 @@ public class ITestOutputStreamSemantics extends AbstractWasbTestBase {
}
}
+ // Block Blobs do not have any StreamCapabilities.
+ @Test
+ public void testBlockBlobCapabilities() throws IOException {
+ Path path = getBlobPathWithTestName(BLOCK_BLOB_DIR);
+ try (FSDataOutputStream stream = fs.create(path)) {
+ assertFalse(stream.hasCapability(StreamCapabilities.HFLUSH));
+ assertFalse(stream.hasCapability(StreamCapabilities.HSYNC));
+ assertFalse(stream.hasCapability(StreamCapabilities.DROPBEHIND));
+ assertFalse(stream.hasCapability(StreamCapabilities.READAHEAD));
+ assertFalse(stream.hasCapability(StreamCapabilities.UNBUFFER));
+ stream.write(getRandomBytes());
+ }
+ }
+
// Verify flush writes data to storage for Block Blobs with compaction
@Test
public void testBlockBlobCompactionFlush() throws Exception {
@@ -347,6 +376,20 @@ public class ITestOutputStreamSemantics extends AbstractWasbTestBase {
}
}
+ // Block Blobs with Compaction have StreamCapabilities.HFLUSH and HSYNC.
+ @Test
+ public void testBlockBlobCompactionCapabilities() throws IOException {
+ Path path = getBlobPathWithTestName(BLOCK_BLOB_COMPACTION_DIR);
+ try (FSDataOutputStream stream = fs.create(path)) {
+ assertTrue(stream.hasCapability(StreamCapabilities.HFLUSH));
+ assertTrue(stream.hasCapability(StreamCapabilities.HSYNC));
+ assertFalse(stream.hasCapability(StreamCapabilities.DROPBEHIND));
+ assertFalse(stream.hasCapability(StreamCapabilities.READAHEAD));
+ assertFalse(stream.hasCapability(StreamCapabilities.UNBUFFER));
+ stream.write(getRandomBytes());
+ }
+ }
+
// A small write does not write data to storage for Page Blobs
@Test
public void testPageBlobSmallWrite() throws IOException {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org