You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by dh...@apache.org on 2008/01/25 07:41:11 UTC

svn commit: r615125 - in /hadoop/core/trunk: CHANGES.txt src/java/org/apache/hadoop/dfs/DFSClient.java src/java/org/apache/hadoop/dfs/DataNode.java src/java/org/apache/hadoop/dfs/FSDataset.java

Author: dhruba
Date: Thu Jan 24 22:41:09 2008
New Revision: 615125

URL: http://svn.apache.org/viewvc?rev=615125&view=rev
Log:
HADOOP-2691. Fix a race condition that was causing the DFSClient
to erroneously remove a good datanode from a pipeline that actually
had another datanode that was bad. (dhruba)


Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java
    hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DataNode.java
    hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSDataset.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=615125&r1=615124&r2=615125&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Jan 24 22:41:09 2008
@@ -557,6 +557,10 @@
     exceptions when servers are restarted by MiniDFSCluster.
     (Sanjay Radia via dhruba)
 
+    HADOOP-2691. Fix a race condition that was causing the DFSClient
+    to erroneously remove a good datanode from a pipeline that actually
+    had another datanode that was bad. (dhruba)
+
 Release 0.15.3 - 2008-01-18
 
   BUG FIXES

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java?rev=615125&r1=615124&r2=615125&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java Thu Jan 24 22:41:09 2008
@@ -1858,9 +1858,9 @@
         nodes = newnodes;
 
         // setup new pipeline
-        success = createBlockOutputStream(nodes, src, true);
         hasError = false;
         errorIndex = 0;
+        success = createBlockOutputStream(nodes, src, true);
       }
 
       response = new ResponseProcessor(nodes);

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DataNode.java?rev=615125&r1=615124&r2=615125&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DataNode.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DataNode.java Thu Jan 24 22:41:09 2008
@@ -1745,14 +1745,16 @@
             // If this is the last packet in block, then close block
             // file and finalize the block before responding success
             if (pkt.lastPacketInBlock) {
-              receiver.close();
-              block.setNumBytes(receiver.offsetInBlock);
-              data.finalizeBlock(block);
-              myMetrics.blocksWritten.inc();
-              notifyNamenodeReceivedBlock(block, EMPTY_DEL_HINT);
-              LOG.info("Received block " + block + 
-                       " of size " + block.getNumBytes() + 
-                       " from " + receiver.inAddr);
+              if (!receiver.finalized) {
+                receiver.close();
+                block.setNumBytes(receiver.offsetInBlock);
+                data.finalizeBlock(block);
+                myMetrics.blocksWritten.inc();
+                notifyNamenodeReceivedBlock(block, EMPTY_DEL_HINT);
+                LOG.info("Received block " + block + 
+                         " of size " + block.getNumBytes() + 
+                         " from " + receiver.inAddr);
+              }
               lastPacket = true;
             }
 
@@ -1828,7 +1830,7 @@
 
             // If this is the last packet in block, then close block
             // file and finalize the block before responding success
-            if (lastPacketInBlock) {
+            if (lastPacketInBlock && !receiver.finalized) {
               receiver.close();
               block.setNumBytes(receiver.offsetInBlock);
               data.finalizeBlock(block);
@@ -1959,15 +1961,17 @@
     // close files
     public void close() throws IOException {
 
-      try {
+      synchronized (currentWriteLock) {
         while (currentWrite) {
-          LOG.info("BlockReceiver for block " + block +
-                   " waiting for last write to drain.");
-          synchronized (currentWriteLock) {
-              currentWriteLock.wait();
+          try {
+            LOG.info("BlockReceiver for block " + block +
+                     " waiting for last write to drain.");
+            currentWriteLock.wait();
+          } catch (InterruptedException e) {
+            throw new IOException("BlockReceiver for block " + block +
+                                  " interrupted drain of last io.");
           }
         }
-      } catch (InterruptedException e) {
       }
       IOException ioe = null;
       // close checksum file
@@ -2044,11 +2048,6 @@
                    block + " to mirror " + mirrorAddr + "\n" +
                    StringUtils.stringifyException(ioe));
           mirrorOut = null;
-          offsetInBlock -= len;
-          synchronized (currentWriteLock) {
-            currentWrite = false;
-            currentWriteLock.notifyAll();
-          }
           //
           // If stream-copy fails, continue
           // writing to disk for replication requests. For client
@@ -2195,8 +2194,10 @@
 
       try {
         // write data chunk header
-        checksumOut.writeShort(FSDataset.METADATA_VERSION);
-        checksum.writeHeader(checksumOut);
+        if (!finalized) {
+          checksumOut.writeShort(FSDataset.METADATA_VERSION);
+          checksum.writeHeader(checksumOut);
+        }
         if (clientName.length() > 0) {
           responder = new Daemon(threadGroup, 
                                  new PacketResponder(this, block, mirrorIn, 
@@ -2275,6 +2276,7 @@
                                 " that is already finalized and is of size " +
                                 data.getLength(block));
         }
+        return;
       }
       if (data.getChannelPosition(block, streams) == offsetInBlock) {
         return;                   // nothing to do 

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSDataset.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSDataset.java?rev=615125&r1=615124&r2=615125&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSDataset.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSDataset.java Thu Jan 24 22:41:09 2008
@@ -610,11 +610,10 @@
       // If the block was succesfully finalized because all packets
       // were successfully processed at the Datanode but the ack for
       // some of the packets were not received by the client. The client 
-      // re-opens the connection and retries sending those packets. The
-      // client will now fail because this datanode has no way of
-      // unfinalizing this block.
+      // re-opens the connection and retries sending those packets.
       // 
-      throw new IOException("Reopen Block " + b + " is valid, and cannot be written to.");
+      DataNode.LOG.info("Reopen Block " + b);
+      return null;
     }
     long blockSize = b.getNumBytes();