You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ki...@apache.org on 2014/09/19 15:52:47 UTC
git commit: HDFS-6948. DN rejects blocks if it has older UC block.
Contributed by Eric Payne. (cherry picked from commit
f02d934fedf00f0ce43d6f3f9b06d89ccc6851a5)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 9333ee306 -> 2a6c9f072
HDFS-6948. DN rejects blocks if it has older UC block. Contributed by
Eric Payne.
(cherry picked from commit f02d934fedf00f0ce43d6f3f9b06d89ccc6851a5)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2a6c9f07
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2a6c9f07
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2a6c9f07
Branch: refs/heads/branch-2
Commit: 2a6c9f0725891f62291c472aeba6b1c3eea21a50
Parents: 9333ee3
Author: Kihwal Lee <ki...@apache.org>
Authored: Fri Sep 19 08:52:16 2014 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Fri Sep 19 08:52:16 2014 -0500
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../datanode/fsdataset/impl/FsDatasetImpl.java | 14 ++++++++---
.../fsdataset/impl/TestWriteToReplica.java | 25 +++++++++++++++++++-
3 files changed, 38 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a6c9f07/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 9bfac36..aeef95a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -227,6 +227,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6970. Move startFile EDEK retries to the DFSClient. (wang)
+ HDFS-6948. DN rejects blocks if it has older UC block
+ (Eric Payne via kihwal)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a6c9f07/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 1584a96..b87552d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -1090,9 +1090,17 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
ExtendedBlock b) throws IOException {
ReplicaInfo replicaInfo = volumeMap.get(b.getBlockPoolId(), b.getBlockId());
if (replicaInfo != null) {
- throw new ReplicaAlreadyExistsException("Block " + b +
- " already exists in state " + replicaInfo.getState() +
- " and thus cannot be created.");
+ if (replicaInfo.getGenerationStamp() < b.getGenerationStamp()
+ && replicaInfo instanceof ReplicaInPipeline) {
+ // Stop the previous writer
+ ((ReplicaInPipeline)replicaInfo)
+ .stopWriter(datanode.getDnConf().getXceiverStopTimeout());
+ invalidate(b.getBlockPoolId(), new Block[]{replicaInfo});
+ } else {
+ throw new ReplicaAlreadyExistsException("Block " + b +
+ " already exists in state " + replicaInfo.getState() +
+ " and thus cannot be created.");
+ }
}
FsVolumeImpl v = volumes.getNextVolume(storageType, b.getNumBytes());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a6c9f07/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
index e6a03d2..a870aa9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestWriteToReplica.java
@@ -111,7 +111,7 @@ public class TestWriteToReplica {
// test writeToTemporary
@Test
- public void testWriteToTempoary() throws Exception {
+ public void testWriteToTemporary() throws Exception {
MiniDFSCluster cluster = new MiniDFSCluster.Builder(new HdfsConfiguration()).build();
try {
cluster.waitActive();
@@ -475,5 +475,28 @@ public class TestWriteToReplica {
}
dataSet.createTemporary(StorageType.DEFAULT, blocks[NON_EXISTENT]);
+
+ try {
+ dataSet.createTemporary(StorageType.DEFAULT, blocks[NON_EXISTENT]);
+ Assert.fail("Should not have created a replica that had already been "
+ + "created " + blocks[NON_EXISTENT]);
+ } catch (Exception e) {
+ Assert.assertTrue(
+ e.getMessage().contains(blocks[NON_EXISTENT].getBlockName()));
+ Assert.assertTrue(e instanceof ReplicaAlreadyExistsException);
+ }
+
+ long newGenStamp = blocks[NON_EXISTENT].getGenerationStamp() * 10;
+ blocks[NON_EXISTENT].setGenerationStamp(newGenStamp);
+ try {
+ ReplicaInPipeline replicaInfo =
+ dataSet.createTemporary(StorageType.DEFAULT, blocks[NON_EXISTENT]);
+ Assert.assertTrue(replicaInfo.getGenerationStamp() == newGenStamp);
+ Assert.assertTrue(
+ replicaInfo.getBlockId() == blocks[NON_EXISTENT].getBlockId());
+ } catch (ReplicaAlreadyExistsException e) {
+ Assert.fail("createRbw() Should have removed the block with the older "
+ + "genstamp and replaced it with the newer one: " + blocks[NON_EXISTENT]);
+ }
}
}