You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cm...@apache.org on 2015/04/24 04:13:36 UTC
hadoop git commit: HDFS-8070. Pre-HDFS-7915 DFSClient cannot use
short circuit on post-HDFS-7915 DataNode (cmccabe)
Repository: hadoop
Updated Branches:
refs/heads/trunk a0e0a6320 -> a8898445d
HDFS-8070. Pre-HDFS-7915 DFSClient cannot use short circuit on post-HDFS-7915 DataNode (cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a8898445
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a8898445
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a8898445
Branch: refs/heads/trunk
Commit: a8898445dc9b5cdb7230e2e23a57393c9f378ff0
Parents: a0e0a63
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Thu Apr 23 19:03:44 2015 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Thu Apr 23 19:13:27 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../apache/hadoop/hdfs/BlockReaderFactory.java | 6 ++-
.../hdfs/protocol/datatransfer/Receiver.java | 3 +-
.../shortcircuit/TestShortCircuitCache.java | 44 ++++++++++++++++++++
4 files changed, 54 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8898445/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 8c3cfe1..b0a0a50 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -589,6 +589,9 @@ Release 2.7.1 - UNRELEASED
HDFS-8147. StorageGroup in Dispatcher should override equals nad hashCode.
(surendra singh lilhore via szetszwo)
+ HDFS-8070. Pre-HDFS-7915 DFSClient cannot use short circuit on
+ post-HDFS-7915 DataNode (cmccabe)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8898445/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java
index 5175a87..714cd68 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderFactory.java
@@ -77,6 +77,9 @@ public class BlockReaderFactory implements ShortCircuitReplicaCreator {
public void injectRequestFileDescriptorsFailure() throws IOException {
// do nothing
}
+ public boolean getSupportsReceiptVerification() {
+ return true;
+ }
}
@VisibleForTesting
@@ -533,7 +536,8 @@ public class BlockReaderFactory implements ShortCircuitReplicaCreator {
final DataOutputStream out =
new DataOutputStream(new BufferedOutputStream(peer.getOutputStream()));
SlotId slotId = slot == null ? null : slot.getSlotId();
- new Sender(out).requestShortCircuitFds(block, token, slotId, 1, true);
+ new Sender(out).requestShortCircuitFds(block, token, slotId, 1,
+ failureInjector.getSupportsReceiptVerification());
DataInputStream in = new DataInputStream(peer.getInputStream());
BlockOpResponseProto resp = BlockOpResponseProto.parseFrom(
PBHelper.vintPrefixed(in));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8898445/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
index 31bdc5e..a6fbb29 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
@@ -186,7 +186,8 @@ public abstract class Receiver implements DataTransferProtocol {
try {
requestShortCircuitFds(PBHelper.convert(proto.getHeader().getBlock()),
PBHelper.convert(proto.getHeader().getToken()),
- slotId, proto.getMaxVersion(), true);
+ slotId, proto.getMaxVersion(),
+ proto.getSupportsReceiptVerification());
} finally {
if (traceScope != null) traceScope.close();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8898445/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
index f3c73f2..20c54fe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
@@ -743,4 +743,48 @@ public class TestShortCircuitCache {
cluster.shutdown();
sockDir.close();
}
+
+ public static class TestPreReceiptVerificationFailureInjector
+ extends BlockReaderFactory.FailureInjector {
+ @Override
+ public boolean getSupportsReceiptVerification() {
+ return false;
+ }
+ }
+
+ // Regression test for HDFS-8070
+ @Test(timeout=60000)
+ public void testPreReceiptVerificationDfsClientCanDoScr() throws Exception {
+ BlockReaderTestUtil.enableShortCircuitShmTracing();
+ TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
+ Configuration conf = createShortCircuitConf(
+ "testPreReceiptVerificationDfsClientCanDoScr", sockDir);
+ conf.setLong(
+ HdfsClientConfigKeys.Read.ShortCircuit.STREAMS_CACHE_EXPIRY_MS_KEY,
+ 1000000000L);
+ MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ cluster.waitActive();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ fs.getClient().getConf().getShortCircuitConf().brfFailureInjector =
+ new TestPreReceiptVerificationFailureInjector();
+ final Path TEST_PATH1 = new Path("/test_file1");
+ DFSTestUtil.createFile(fs, TEST_PATH1, 4096, (short)1, 0xFADE2);
+ final Path TEST_PATH2 = new Path("/test_file2");
+ DFSTestUtil.createFile(fs, TEST_PATH2, 4096, (short)1, 0xFADE2);
+ DFSTestUtil.readFileBuffer(fs, TEST_PATH1);
+ DFSTestUtil.readFileBuffer(fs, TEST_PATH2);
+ ShortCircuitRegistry registry =
+ cluster.getDataNodes().get(0).getShortCircuitRegistry();
+ registry.visit(new ShortCircuitRegistry.Visitor() {
+ @Override
+ public void accept(HashMap<ShmId, RegisteredShm> segments,
+ HashMultimap<ExtendedBlockId, Slot> slots) {
+ Assert.assertEquals(1, segments.size());
+ Assert.assertEquals(2, slots.size());
+ }
+ });
+ cluster.shutdown();
+ sockDir.close();
+ }
}