You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2015/09/02 00:20:53 UTC
[4/4] hadoop git commit: HDFS-7885. Datanode should not trust the
generation stamp provided by client. Contributed by Tsz Wo Nicholas Sze.
HDFS-7885. Datanode should not trust the generation stamp provided by client. Contributed by Tsz Wo Nicholas Sze.
(cherry picked from commit 24db0812be64e83a48ade01fc1eaaeaedad4dec0)
(cherry picked from commit 994dadb9ba0a3b87b6548e6e0801eadd26554d55)
(cherry picked from commit 0bc5c6495a7feb4365af0ce5fe48fc87b7e1749f)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b25491dc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b25491dc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b25491dc
Branch: refs/heads/branch-2.6.1
Commit: b25491dc45be92cc3ec157d998a2632387c3952f
Parents: 7ffdf7d
Author: Jing Zhao <ji...@apache.org>
Authored: Fri Mar 6 10:55:56 2015 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Tue Sep 1 15:17:57 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../datanode/fsdataset/impl/FsDatasetImpl.java | 15 +++++
.../hadoop/hdfs/TestBlockReaderLocalLegacy.java | 63 ++++++++++++++++++++
3 files changed, 81 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b25491dc/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1811337..d7ff237 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -85,6 +85,9 @@ Release 2.6.1 - UNRELEASED
HDFS-7871. NameNodeEditLogRoller can keep printing "Swallowing exception"
message. (jing9)
+ HDFS-7885. Datanode should not trust the generation stamp provided by
+ client. (Tsz Wo Nicholas Sze via jing9)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b25491dc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 0d9f096..0c2337e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -2276,6 +2276,21 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
@Override // FsDatasetSpi
public BlockLocalPathInfo getBlockLocalPathInfo(ExtendedBlock block)
throws IOException {
+ synchronized(this) {
+ final Replica replica = volumeMap.get(block.getBlockPoolId(),
+ block.getBlockId());
+ if (replica == null) {
+ throw new ReplicaNotFoundException(block);
+ }
+ if (replica.getGenerationStamp() < block.getGenerationStamp()) {
+ throw new IOException(
+ "Replica generation stamp < block generation stamp, block="
+ + block + ", replica=" + replica);
+ } else if (replica.getGenerationStamp() > block.getGenerationStamp()) {
+ block.setGenerationStamp(replica.getGenerationStamp());
+ }
+ }
+
File datafile = getBlockFile(block);
File metafile = FsDatasetUtil.getMetaFile(datafile, block.getGenerationStamp());
BlockLocalPathInfo info = new BlockLocalPathInfo(block,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b25491dc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocalLegacy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocalLegacy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocalLegacy.java
index cb50539..1c4134f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocalLegacy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocalLegacy.java
@@ -30,11 +30,16 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
+import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.unix.DomainSocket;
import org.apache.hadoop.net.unix.TemporarySocketDirectory;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.BeforeClass;
@@ -153,4 +158,62 @@ public class TestBlockReaderLocalLegacy {
Arrays.equals(orig, buf);
cluster.shutdown();
}
+
+ @Test(timeout=20000)
+ public void testBlockReaderLocalLegacyWithAppend() throws Exception {
+ final short REPL_FACTOR = 1;
+ final HdfsConfiguration conf = getConfiguration(null);
+ conf.setBoolean(DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL, true);
+
+ final MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ cluster.waitActive();
+
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+ final Path path = new Path("/testBlockReaderLocalLegacy");
+ DFSTestUtil.createFile(dfs, path, 10, REPL_FACTOR, 0);
+ DFSTestUtil.waitReplication(dfs, path, REPL_FACTOR);
+
+ final ClientDatanodeProtocol proxy;
+ final Token<BlockTokenIdentifier> token;
+ final ExtendedBlock originalBlock;
+ final long originalGS;
+ {
+ final LocatedBlock lb = cluster.getNameNode().getRpcServer()
+ .getBlockLocations(path.toString(), 0, 1).get(0);
+ proxy = DFSUtil.createClientDatanodeProtocolProxy(
+ lb.getLocations()[0], conf, 60000, false);
+ token = lb.getBlockToken();
+
+ // get block and generation stamp
+ final ExtendedBlock blk = new ExtendedBlock(lb.getBlock());
+ originalBlock = new ExtendedBlock(blk);
+ originalGS = originalBlock.getGenerationStamp();
+
+ // test getBlockLocalPathInfo
+ final BlockLocalPathInfo info = proxy.getBlockLocalPathInfo(blk, token);
+ Assert.assertEquals(originalGS, info.getBlock().getGenerationStamp());
+ }
+
+ { // append one byte
+ FSDataOutputStream out = dfs.append(path);
+ out.write(1);
+ out.close();
+ }
+
+ {
+ // get new generation stamp
+ final LocatedBlock lb = cluster.getNameNode().getRpcServer()
+ .getBlockLocations(path.toString(), 0, 1).get(0);
+ final long newGS = lb.getBlock().getGenerationStamp();
+ Assert.assertTrue(newGS > originalGS);
+
+ // getBlockLocalPathInfo using the original block.
+ Assert.assertEquals(originalGS, originalBlock.getGenerationStamp());
+ final BlockLocalPathInfo info = proxy.getBlockLocalPathInfo(
+ originalBlock, token);
+ Assert.assertEquals(newGS, info.getBlock().getGenerationStamp());
+ }
+ cluster.shutdown();
+ }
}