You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2018/03/30 23:31:06 UTC
[2/2] hadoop git commit: HDFS-12299. Race Between update pipeline and
DN Re-Registration. Contributed by Brahma Reddy Battula.
HDFS-12299. Race Between update pipeline and DN Re-Registration. Contributed by Brahma Reddy Battula.
(cherry picked from commit 5a83ffa396089972e23c533eca33c9cba231c45a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4a5d0e5a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4a5d0e5a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4a5d0e5a
Branch: refs/heads/branch-2.7
Commit: 4a5d0e5a4c918ec4d370c21fee39733e219e8948
Parents: 257d5a1
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Fri Mar 30 16:29:31 2018 -0700
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Fri Mar 30 16:29:31 2018 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 15 ++++++
.../BlockInfoContiguousUnderConstruction.java | 5 +-
.../TestClientProtocolForPipelineRecovery.java | 50 ++++++++++++++++++++
.../namenode/ha/TestRetryCacheWithHA.java | 9 +++-
5 files changed, 79 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a5d0e5a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 6ad1070..d856310 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -45,6 +45,9 @@ Release 2.7.6 - UNRELEASED
HDFS-13195. DataNode conf page cannot display the current value after
reconfig. (Mao Baolong via kihwal)
+ HDFS-12299. Race Between update pipeline and DN Re-Registration.
+ (Brahma Reddy Battula. Backport by Wei-Chiu Chuang, Konstantin Shvachko)
+
Release 2.7.5 - 2017-12-14
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a5d0e5a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index 2bf4e1b..89df16c 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -1345,6 +1345,21 @@ public class DFSOutputStream extends FSOutputSummer
return false; // do not sleep, continue processing
}
+ void updateBlockGS(final long newGS) {
+ block.setGenerationStamp(newGS);
+ }
+
+ /** update pipeline at the namenode */
+ @VisibleForTesting
+ public void updatePipeline(long newGS) throws IOException {
+ final ExtendedBlock oldBlock = block.getCurrentBlock();
+ // the new GS has been propagated to all DN, it should be ok to update the
+ // local block state
+ updateBlockGS(newGS);
+ dfsClient.namenode.updatePipeline(dfsClient.clientName, oldBlock,
+ block.getCurrentBlock(), nodes, storageIDs);
+ }
+
DatanodeInfo[] getExcludedNodes() {
return excludedNodes.getAllPresent(excludedNodes.asMap().keySet())
.keySet().toArray(new DatanodeInfo[0]);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a5d0e5a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
index 409295a..e98ab7e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
@@ -193,8 +193,9 @@ public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous {
int numLocations = targets == null ? 0 : targets.length;
this.replicas = new ArrayList<ReplicaUnderConstruction>(numLocations);
for(int i = 0; i < numLocations; i++)
- replicas.add(
- new ReplicaUnderConstruction(this, targets[i], ReplicaState.RBW));
+ if (targets[i] != null) {
+ replicas.add(new ReplicaUnderConstruction(this, targets[i], ReplicaState.RBW));
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a5d0e5a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
index 63a6f62..29e3c8b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
@@ -32,6 +32,9 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeFaultInjector;
import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException;
@@ -527,4 +530,51 @@ public class TestClientProtocolForPipelineRecovery {
}
}
}
+
+ @Test
+ public void testUpdatePipeLineAfterDNReg()throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
+ cluster.waitActive();
+ FileSystem fileSys = cluster.getFileSystem();
+
+ Path file = new Path("/testUpdatePipeLineAfterDNReg");
+ FSDataOutputStream out = fileSys.create(file);
+ out.write(1);
+ out.hflush();
+ //Get the First DN and disable the heartbeats and then put in Deadstate
+ DFSOutputStream dfsOut = (DFSOutputStream) out.getWrappedStream();
+ DatanodeInfo[] pipeline = dfsOut.getPipeline();
+ DataNode dn1 = cluster.getDataNode(pipeline[0].getIpcPort());
+ dn1.setHeartbeatsDisabledForTests(true);
+ DatanodeDescriptor dn1Desc = cluster.getNamesystem(0).getBlockManager()
+ .getDatanodeManager().getDatanode(dn1.getDatanodeId());
+ cluster.setDataNodeDead(dn1Desc);
+ //Re-register the DeadNode
+ DatanodeProtocolClientSideTranslatorPB dnp =
+ new DatanodeProtocolClientSideTranslatorPB(
+ cluster.getNameNode().getNameNodeAddress(), conf);
+ dnp.registerDatanode(
+ dn1.getDNRegistrationForBP(cluster.getNamesystem().getBlockPoolId()));
+ DFSOutputStream dfsO = (DFSOutputStream) out.getWrappedStream();
+ String clientName = ((DistributedFileSystem) fileSys).getClient()
+ .getClientName();
+ NamenodeProtocols namenode = cluster.getNameNodeRpc();
+ //Update the genstamp and call updatepipeline
+ LocatedBlock newBlock = namenode
+ .updateBlockForPipeline(dfsO.getBlock(), clientName);
+ dfsO.getStreamer()
+ .updatePipeline(newBlock.getBlock().getGenerationStamp());
+ newBlock = namenode.updateBlockForPipeline(dfsO.getBlock(), clientName);
+ //Should not throw any error Pipeline should be success
+ dfsO.getStreamer()
+ .updatePipeline(newBlock.getBlock().getGenerationStamp());
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a5d0e5a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
index 1c270aa..6c96991 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
@@ -72,6 +72,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper;
@@ -735,7 +736,13 @@ public class TestRetryCacheWithHA {
DatanodeInfo[] newNodes = new DatanodeInfo[2];
newNodes[0] = nodes[0];
newNodes[1] = nodes[1];
- String[] storageIDs = {"s0", "s1"};
+ final DatanodeManager dm = cluster.getNamesystem(0).getBlockManager()
+ .getDatanodeManager();
+ final String storageID1 = dm.getDatanode(newNodes[0]).getStorageInfos()[0]
+ .getStorageID();
+ final String storageID2 = dm.getDatanode(newNodes[1]).getStorageInfos()[0]
+ .getStorageID();
+ String[] storageIDs = {storageID1, storageID2};
client.getNamenode().updatePipeline(client.getClientName(), oldBlock,
newBlock, newNodes, storageIDs);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org