You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2015/03/09 21:26:03 UTC
[37/50] [abbrv] hadoop git commit: HDFS-7885. Datanode should not
trust the generation stamp provided by client. Contributed by Tsz Wo Nicholas
Sze.
HDFS-7885. Datanode should not trust the generation stamp provided by client. Contributed by Tsz Wo Nicholas Sze.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/055267d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/055267d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/055267d5
Branch: refs/heads/HDFS-7285
Commit: 055267d50cffb51f28f271f27016df23fae2d222
Parents: 7070347
Author: Jing Zhao <ji...@apache.org>
Authored: Fri Mar 6 10:55:56 2015 -0800
Committer: Jing Zhao <ji...@apache.org>
Committed: Mon Mar 9 13:11:26 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../datanode/fsdataset/impl/FsDatasetImpl.java | 15 +++++
.../hadoop/hdfs/TestBlockReaderLocalLegacy.java | 63 ++++++++++++++++++++
3 files changed, 81 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/055267d5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 763d327..e622a57 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1104,6 +1104,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7434. DatanodeID hashCode should not be mutable. (daryn via kihwal)
+ HDFS-7885. Datanode should not trust the generation stamp provided by
+ client. (Tsz Wo Nicholas Sze via jing9)
+
BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS
HDFS-7720. Quota by Storage Type API, tools and ClientNameNode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/055267d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index cc6220a..58f5615 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -2568,6 +2568,21 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
@Override // FsDatasetSpi
public BlockLocalPathInfo getBlockLocalPathInfo(ExtendedBlock block)
throws IOException {
+ synchronized(this) {
+ final Replica replica = volumeMap.get(block.getBlockPoolId(),
+ block.getBlockId());
+ if (replica == null) {
+ throw new ReplicaNotFoundException(block);
+ }
+ if (replica.getGenerationStamp() < block.getGenerationStamp()) {
+ throw new IOException(
+ "Replica generation stamp < block generation stamp, block="
+ + block + ", replica=" + replica);
+ } else if (replica.getGenerationStamp() > block.getGenerationStamp()) {
+ block.setGenerationStamp(replica.getGenerationStamp());
+ }
+ }
+
File datafile = getBlockFile(block);
File metafile = FsDatasetUtil.getMetaFile(datafile, block.getGenerationStamp());
BlockLocalPathInfo info = new BlockLocalPathInfo(block,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/055267d5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocalLegacy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocalLegacy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocalLegacy.java
index cb50539..1c4134f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocalLegacy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocalLegacy.java
@@ -30,11 +30,16 @@ import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo;
+import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.unix.DomainSocket;
import org.apache.hadoop.net.unix.TemporarySocketDirectory;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.BeforeClass;
@@ -153,4 +158,62 @@ public class TestBlockReaderLocalLegacy {
Arrays.equals(orig, buf);
cluster.shutdown();
}
+
+ @Test(timeout=20000)
+ public void testBlockReaderLocalLegacyWithAppend() throws Exception {
+ final short REPL_FACTOR = 1;
+ final HdfsConfiguration conf = getConfiguration(null);
+ conf.setBoolean(DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKREADERLOCAL, true);
+
+ final MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ cluster.waitActive();
+
+ final DistributedFileSystem dfs = cluster.getFileSystem();
+ final Path path = new Path("/testBlockReaderLocalLegacy");
+ DFSTestUtil.createFile(dfs, path, 10, REPL_FACTOR, 0);
+ DFSTestUtil.waitReplication(dfs, path, REPL_FACTOR);
+
+ final ClientDatanodeProtocol proxy;
+ final Token<BlockTokenIdentifier> token;
+ final ExtendedBlock originalBlock;
+ final long originalGS;
+ {
+ final LocatedBlock lb = cluster.getNameNode().getRpcServer()
+ .getBlockLocations(path.toString(), 0, 1).get(0);
+ proxy = DFSUtil.createClientDatanodeProtocolProxy(
+ lb.getLocations()[0], conf, 60000, false);
+ token = lb.getBlockToken();
+
+ // get block and generation stamp
+ final ExtendedBlock blk = new ExtendedBlock(lb.getBlock());
+ originalBlock = new ExtendedBlock(blk);
+ originalGS = originalBlock.getGenerationStamp();
+
+ // test getBlockLocalPathInfo
+ final BlockLocalPathInfo info = proxy.getBlockLocalPathInfo(blk, token);
+ Assert.assertEquals(originalGS, info.getBlock().getGenerationStamp());
+ }
+
+ { // append one byte
+ FSDataOutputStream out = dfs.append(path);
+ out.write(1);
+ out.close();
+ }
+
+ {
+ // get new generation stamp
+ final LocatedBlock lb = cluster.getNameNode().getRpcServer()
+ .getBlockLocations(path.toString(), 0, 1).get(0);
+ final long newGS = lb.getBlock().getGenerationStamp();
+ Assert.assertTrue(newGS > originalGS);
+
+ // getBlockLocalPathInfo using the original block.
+ Assert.assertEquals(originalGS, originalBlock.getGenerationStamp());
+ final BlockLocalPathInfo info = proxy.getBlockLocalPathInfo(
+ originalBlock, token);
+ Assert.assertEquals(newGS, info.getBlock().getGenerationStamp());
+ }
+ cluster.shutdown();
+ }
}