svn commit: r637724 - in /hadoop/core/trunk: CHANGES.txt src/java/org/apache/hadoop/dfs/FSNamesystem.java

[dh...@apache.org]

Author: dhruba
Date: Sun Mar 16 22:24:19 2008
New Revision: 637724

URL: http://svn.apache.org/viewvc?rev=637724&view=rev
Log:
HADOOP-2890. If different datanodes report the same block but
with different sizes to the namenode, the namenode picks the
replica(s) with the largest size as the only valid replica(s). (dhruba)


Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=637724&r1=637723&r2=637724&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Sun Mar 16 22:24:19 2008
@@ -237,6 +237,9 @@
     bugs in JSPs to do with analysis - HADOOP-2742, HADOOP-2792.
     (Amareshwari Sri Ramadasu via ddas)
 
+    HADOOP-2890. If different datanodes report the same block but
+    with different sizes to the namenode, the namenode picks the
+    replica(s) with the largest size as the only valid replica(s). (dhruba)
 
 Release 0.16.1 - 2008-03-13
 

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java?rev=637724&r1=637723&r2=637724&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java Sun Mar 16 22:24:19 2008
@@ -2589,9 +2589,45 @@
                    " reported from " + node.getName() + 
                    " current size is " + cursize +
                    " reported size is " + block.getNumBytes());
-          // Accept this block even if there is a problem with its
-          // size. Clients should detect data corruption because of
-          // CRC mismatch.
+          try {
+            if (cursize > block.getNumBytes()) {
+              // new replica is smaller in size than existing block.
+              // Delete new replica.
+              LOG.warn("Deleting block " + block + " from " + node.getName());
+              invalidateBlock(block, node);
+            } else {
+              // new replica is larger in size than existing block.
+              // Delete pre-existing replicas.
+              int numNodes = blocksMap.numNodes(block);
+              int count = 0;
+              DatanodeDescriptor nodes[] = new DatanodeDescriptor[numNodes];
+              Iterator<DatanodeDescriptor> it = blocksMap.nodeIterator(block);
+              for (; it != null && it.hasNext(); ) {
+                DatanodeDescriptor dd = it.next();
+                if (!dd.equals(node)) {
+                  nodes[count++] = dd;
+                }
+              }
+              for (int j = 0; j < count; j++) {
+                LOG.warn("Deleting block " + block + " from " + 
+                         nodes[j].getName());
+                invalidateBlock(block, nodes[j]);
+              }
+              //
+              // change the size of block in blocksMap
+              //
+              storedBlock = blocksMap.getStoredBlock(block); //extra look up!
+              if (storedBlock == null) {
+                LOG.warn("Block " + block + 
+                   " reported from " + node.getName() + 
+                   " does not exist in blockMap. Surprise! Surprise!");
+              } else {
+                storedBlock.setNumBytes(block.getNumBytes());
+              }
+            }
+          } catch (IOException e) {
+            LOG.warn("Error in deleting bad block " + block + e);
+          }
         }
       }
       block = storedBlock;